@lightdash/warehouses 3.127.1 → 3.128.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo.build +1 -1
- package/dist/warehouseClientFromCredentials.d.ts +2 -1
- package/dist/warehouseClientFromCredentials.d.ts.map +1 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.d.ts +16 -4
- package/dist/warehouseClients/BigqueryWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.js +33 -12
- package/package.json +2 -2
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { CreateWarehouseCredentials } from '@lightdash/common';
|
|
2
2
|
import { WarehouseClient } from './types';
|
|
3
3
|
import { type AthenaWarehouseClientOptions } from './warehouseClients/AthenaWarehouseClient';
|
|
4
|
+
import { type BigqueryWarehouseClientOptions } from './warehouseClients/BigqueryWarehouseClient';
|
|
4
5
|
import { type ClickhouseWarehouseClientOptions } from './warehouseClients/ClickhouseWarehouseClient';
|
|
5
6
|
import { type DuckdbWarehouseClientOptions } from './warehouseClients/DuckdbWarehouseClient';
|
|
6
|
-
export type WarehouseClientOptions = DuckdbWarehouseClientOptions & ClickhouseWarehouseClientOptions & AthenaWarehouseClientOptions & {
|
|
7
|
+
export type WarehouseClientOptions = DuckdbWarehouseClientOptions & ClickhouseWarehouseClientOptions & AthenaWarehouseClientOptions & BigqueryWarehouseClientOptions & {
|
|
7
8
|
agentSession?: boolean;
|
|
8
9
|
agentJobControls?: boolean;
|
|
9
10
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"warehouseClientFromCredentials.d.ts","sourceRoot":"","sources":["../src/warehouseClientFromCredentials.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,0BAA0B,EAG7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAEH,KAAK,4BAA4B,EACpC,MAAM,0CAA0C,CAAC;
|
|
1
|
+
{"version":3,"file":"warehouseClientFromCredentials.d.ts","sourceRoot":"","sources":["../src/warehouseClientFromCredentials.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,0BAA0B,EAG7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAEH,KAAK,4BAA4B,EACpC,MAAM,0CAA0C,CAAC;AAClD,OAAO,EAEH,KAAK,8BAA8B,EACtC,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAEH,KAAK,gCAAgC,EACxC,MAAM,8CAA8C,CAAC;AAEtD,OAAO,EAEH,KAAK,4BAA4B,EACpC,MAAM,0CAA0C,CAAC;AAMlD,MAAM,MAAM,sBAAsB,GAAG,4BAA4B,GAC7D,gCAAgC,GAChC,4BAA4B,GAC5B,8BAA8B,GAAG;IAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEN,eAAO,MAAM,8BAA8B,gBAC1B,0BAA0B,YAC7B,sBAAsB,KACjC,eA4BF,CAAC"}
|
|
@@ -60,16 +60,28 @@ type GoogleOauthTokenError = {
|
|
|
60
60
|
errorSubtype: string | undefined;
|
|
61
61
|
};
|
|
62
62
|
export declare const getGoogleOauthTokenError: (error: unknown) => GoogleOauthTokenError | undefined;
|
|
63
|
+
export type BigqueryExternalAccountCredentials = {
|
|
64
|
+
type: 'external_account';
|
|
65
|
+
audience: string;
|
|
66
|
+
subject_token_type: 'urn:ietf:params:oauth:token-type:jwt';
|
|
67
|
+
token_url: 'https://sts.googleapis.com/v1/token';
|
|
68
|
+
subject_token_supplier: {
|
|
69
|
+
getSubjectToken: () => Promise<string>;
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
export type BigqueryWarehouseClientOptions = {
|
|
73
|
+
bigqueryExternalAccount?: BigqueryExternalAccountCredentials;
|
|
74
|
+
agentSession?: boolean;
|
|
75
|
+
agentJobControls?: boolean;
|
|
76
|
+
};
|
|
63
77
|
export declare class BigqueryWarehouseClient extends WarehouseBaseClient<CreateBigqueryCredentials> {
|
|
64
78
|
private static readonly MAX_LABELS;
|
|
65
79
|
private static readonly AGENT_IDENTITY_LABEL_KEYS;
|
|
66
80
|
private static readonly PRIORITY_LABEL_KEYS;
|
|
67
81
|
client: BigQuery;
|
|
68
82
|
private readonly agentJobControls;
|
|
69
|
-
constructor(credentials: CreateBigqueryCredentials, options?:
|
|
70
|
-
|
|
71
|
-
agentJobControls?: boolean;
|
|
72
|
-
});
|
|
83
|
+
constructor(credentials: CreateBigqueryCredentials, options?: BigqueryWarehouseClientOptions);
|
|
84
|
+
private static getAuthenticationOptions;
|
|
73
85
|
static isBigqueryError(error: unknown): error is BigqueryError;
|
|
74
86
|
private usesUserRefreshToken;
|
|
75
87
|
private translateGoogleOauthTokenError;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,
|
|
1
|
+
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EAEpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAIP,eAAe,EACf,eAAe,EACf,6BAA6B,EAE7B,yBAAyB,EAMzB,MAAM,EAEN,eAAe,EAMf,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,aAAa,EAEb,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAE5B,KAAK,sBAAsB,EAC9B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACH,gBAAgB,EAChB,0BAA0B,EAC1B,8BAA8B,EAEjC,MAAM,UAAU,CAAC;AAOlB,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,iBAAiB;IACzB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,OAAO,YAAY;IACnB,UAAU,eAAe;IACzB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,KAAK,UAAU;CAClB;AAqFD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,GAAG,SAAS,KACzB,eAAe,GAAG,SASpB,CAAC;AA2BF,eAAO,MAAM,sBAAsB,UAAW;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,KAAG,iBAAiB,GAAG,IAuBvB,CAAC;AAEF,KAAK,WAAW,GAAG;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;CAC1B,CAAC;AAEF,KAAK,YAAY,GAAG,QAAQ,CACxB,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM,CAAC,CACpD,GACG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC;AAkDxD,KAAK,aAAa,GAAG;IACjB,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;CAClC,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,QAAQ,CAAC,IAAI,2BAA2B;IAExC,cAAc,IAAI,mBAAmB,CAEpC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAWhD;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGlC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAG7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGvC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAM1D;IAED,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,CAE3C;CACJ;AAID,KAAK,qBAAqB,GAAG;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAUF,eAAO,MAAM,wBAAwB,UAC1B,OAAO,KACf,qBAAqB,GAAG,SAe1B,CAAC;AAEF,MAAM,MAAM,kCAAkC,GAAG;IAC7C,IAAI,EAAE,kBAAkB,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,kBAAkB,EAAE,sCAAsC,CAAC;IAC3D,SAAS,EAAE,qCAAqC,CAAC;IACjD,sBAAsB,EAAE;QAAE,eAAe,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAA;KAAE,CAAC;CACtE,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG;IACzC,uBAAuB,CAAC,EAAE,kCAAkC,CAAC;IAC7D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEF,qBAAa,uBAAwB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACvF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAM;IAExC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAK/C;IAGF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAGzC;IAEF,MAAM,EAAE,QAAQ,CAAC;IAEjB,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;IAE3C,YACI,WAAW,EAAE,yBAAyB,EACtC,OAAO,CAAC,EAAE,8BAA8B,EA4B3C;IAED,OAAO,CAAC,MAAM,CAAC,wBAAwB;IAgCvC,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAE7D;IAED,OAAO,CAAC,oBAAoB;IAmB5B,OAAO,CAAC,8BAA8B;IAsBtC,OAAO,CAAC,4BAA4B;IAOpC;;;;;;;;;;OAUG;IACH,MAAM,CAAC,wBAAwB,CAC3B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CA0BpC;IAED,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,SAAS,oEAgBtE;YAEa,SAAS;YAoET,MAAM;IAWpB,MAAM,CAAC,iBAAiB,CACpB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,UAAU,EAAE,QAAQ,CAAC,cAAc,GAAG,SAAS,GAChD,uBAAuB,CAqBzB;YAEa,qBAAqB;YAUrB,aAAa;IAoCrB,WAAW,CACb,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC/B,GACF,OAAO,CAAC,IAAI,CAAC,CAuBf;IAED,OAAa,gBAAgB,CACzB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAShD;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,6BA2EN;IAEK,YAAY;;;;;;SAOjB;YAEa,sBAAsB;IAgE9B,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAmC3B;IAED,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,GAAE,MAAW,uBA+FzD;IAEK,aAAa,CAAC,EAChB,GAAG,EACH,MAAM,EACN,IAAI,EACJ,QAAQ,GACX,EAAE,8BAA8B,GAAG,OAAO,CAAC,sBAAsB,CAAC,CA2BlE;IAEK,iBAAiB,CACnB,EACI,GAAG,EACH,MAAM,EACN,IAAI,EACJ,QAAQ,EACR,kBAAkB,GACrB,EAAE,8BAA8B,EACjC,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GACV,OAAO,CAAC,0BAA0B,CAAC,CAoErC;YAEa,kBAAkB;IAWhC,OAAa,YAAY,CACrB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,cAAc,GAAE;QACZ,WAAW,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KASvD,GACH,OAAO,CAAC,eAAe,EAAE,CAAC,CA0B5B;IAED,OAAa,wBAAwB,CACjC,QAAQ,EAAE,eAAe,EAAE,EAC3B,YAAY,EAAE,MAAM,EACpB,qBAAqB,GAAE,CAAC,SAAS,EAAE,MAAM,KAAK;QAC1C,WAAW,EAAE,CACT,OAAO,EAAE,kBAAkB,KAC1B,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/B,KAAK,EAAE,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAgBpC,GACF,OAAO,CAAC,6BAA6B,CAAC,CAuFxC;IAED,OAAa,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA0DxE;CACJ"}
|
|
@@ -293,28 +293,49 @@ class BigqueryWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
293
293
|
location: credentials.location || undefined,
|
|
294
294
|
maxRetries: credentials.retries,
|
|
295
295
|
apiEndpoint: credentials.accessUrl || undefined,
|
|
296
|
-
...(credentials
|
|
297
|
-
common_1.BigqueryAuthenticationType.ADC
|
|
298
|
-
? {
|
|
299
|
-
// Support ADC via workforce identity federation / external_account configuration.
|
|
300
|
-
// In this case we should rely on ADC at runtime and not pass explicit credentials.
|
|
301
|
-
}
|
|
302
|
-
: {
|
|
303
|
-
credentials: (0, common_1.getBigqueryKeyfileCredentials)(credentials.keyfileContents),
|
|
304
|
-
}),
|
|
296
|
+
...BigqueryWarehouseClient.getAuthenticationOptions(credentials, options),
|
|
305
297
|
});
|
|
306
298
|
}
|
|
307
299
|
catch (e) {
|
|
308
300
|
throw new common_1.WarehouseConnectionError(`Failed connection to ${credentials.project} in ${credentials.location}. ${(0, common_1.getErrorMessage)(e)}`);
|
|
309
301
|
}
|
|
310
302
|
}
|
|
303
|
+
static getAuthenticationOptions(credentials, options) {
|
|
304
|
+
switch (credentials.authenticationType) {
|
|
305
|
+
case common_1.BigqueryAuthenticationType.ADC:
|
|
306
|
+
// Support ADC via workforce identity federation / external_account configuration.
|
|
307
|
+
// In this case we should rely on ADC at runtime and not pass explicit credentials.
|
|
308
|
+
return {};
|
|
309
|
+
case common_1.BigqueryAuthenticationType.WORKLOAD_IDENTITY_FEDERATION:
|
|
310
|
+
if (!options?.bigqueryExternalAccount) {
|
|
311
|
+
throw new common_1.WarehouseConnectionError('BigQuery workload identity federation credentials are unavailable.');
|
|
312
|
+
}
|
|
313
|
+
return { credentials: options.bigqueryExternalAccount };
|
|
314
|
+
case common_1.BigqueryAuthenticationType.PRIVATE_KEY:
|
|
315
|
+
case common_1.BigqueryAuthenticationType.SSO:
|
|
316
|
+
case undefined:
|
|
317
|
+
return {
|
|
318
|
+
credentials: (0, common_1.getBigqueryKeyfileCredentials)(credentials.keyfileContents),
|
|
319
|
+
};
|
|
320
|
+
default:
|
|
321
|
+
return (0, common_1.assertUnreachable)(credentials.authenticationType, 'Unknown BigQuery authentication type');
|
|
322
|
+
}
|
|
323
|
+
}
|
|
311
324
|
static isBigqueryError(error) {
|
|
312
325
|
return error !== null && typeof error === 'object' && 'errors' in error;
|
|
313
326
|
}
|
|
314
327
|
usesUserRefreshToken() {
|
|
315
|
-
|
|
316
|
-
common_1.BigqueryAuthenticationType.ADC
|
|
317
|
-
|
|
328
|
+
switch (this.credentials.authenticationType) {
|
|
329
|
+
case common_1.BigqueryAuthenticationType.ADC:
|
|
330
|
+
case common_1.BigqueryAuthenticationType.WORKLOAD_IDENTITY_FEDERATION:
|
|
331
|
+
return false;
|
|
332
|
+
case common_1.BigqueryAuthenticationType.PRIVATE_KEY:
|
|
333
|
+
case common_1.BigqueryAuthenticationType.SSO:
|
|
334
|
+
case undefined:
|
|
335
|
+
return (this.credentials.keyfileContents?.type === 'authorized_user');
|
|
336
|
+
default:
|
|
337
|
+
return (0, common_1.assertUnreachable)(this.credentials.authenticationType, 'Unknown BigQuery authentication type');
|
|
338
|
+
}
|
|
318
339
|
}
|
|
319
340
|
translateGoogleOauthTokenError(error) {
|
|
320
341
|
const tokenError = (0, exports.getGoogleOauthTokenError)(error);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.128.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "3.
|
|
24
|
+
"@lightdash/common": "3.128.1",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|