@lightdash/warehouses 2.584.0 → 2.586.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { BigQuery, Dataset, DatasetsResponse, GetDatasetsOptions, QueryRowsResponse } from '@google-cloud/bigquery';
|
|
2
2
|
import bigquery from '@google-cloud/bigquery/build/src/types';
|
|
3
|
-
import { AnyType, BigqueryDataset, BigqueryProject, BigqueryProjectRecommendation, CreateBigqueryCredentials, Metric, PartitionColumn, SupportedDbtAdapter, TimeIntervalUnit, UnnestSql, UnnestSqlArgs, WarehouseQueryError, WarehouseResults, WarehouseTypes, type ResultNumericKind, type TimestampDomain, type WarehouseExecutionStats } from '@lightdash/common';
|
|
3
|
+
import { AnyType, BigqueryDataset, BigqueryProject, BigqueryProjectRecommendation, CreateBigqueryCredentials, Metric, PartitionColumn, SupportedDbtAdapter, TimeIntervalUnit, UnnestSql, UnnestSqlArgs, WarehouseQueryError, WarehouseResults, WarehouseTypes, type ResultNumericKind, type TimestampDomain, type WarehouseExecutionStats, type WarehouseQueryEstimate } from '@lightdash/common';
|
|
4
4
|
import { WarehouseCatalog, WarehouseExecuteAsyncQuery, WarehouseExecuteAsyncQueryArgs } from '../types';
|
|
5
5
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
6
6
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
@@ -96,6 +96,7 @@ export declare class BigqueryWarehouseClient extends WarehouseBaseClient<CreateB
|
|
|
96
96
|
values?: AnyType[];
|
|
97
97
|
tags?: Record<string, string>;
|
|
98
98
|
timezone?: string;
|
|
99
|
+
maximumBytesBilled?: number;
|
|
99
100
|
}): Promise<void>;
|
|
100
101
|
static getTableMetadata(dataset: Dataset, table: string): Promise<[string, string, string, TableSchema]>;
|
|
101
102
|
getCatalog(requests: {
|
|
@@ -113,7 +114,8 @@ export declare class BigqueryWarehouseClient extends WarehouseBaseClient<CreateB
|
|
|
113
114
|
private getAllTablesFromClient;
|
|
114
115
|
getFields(tableName: string, schema: string, database?: string): Promise<WarehouseCatalog>;
|
|
115
116
|
parseError(error: bigquery.IErrorProto, query?: string): WarehouseQueryError;
|
|
116
|
-
|
|
117
|
+
estimateQuery({ sql, values, tags, timezone, }: WarehouseExecuteAsyncQueryArgs): Promise<WarehouseQueryEstimate>;
|
|
118
|
+
executeAsyncQuery({ sql, values, tags, timezone, maximumBytesBilled, }: WarehouseExecuteAsyncQueryArgs, resultsStreamCallback?: (rows: WarehouseResults['rows'], fields: WarehouseResults['fields']) => void): Promise<WarehouseExecuteAsyncQuery>;
|
|
117
119
|
private awaitJobCompletion;
|
|
118
120
|
static getDatabases(projectId: string, refresh_token: string, bigqueryClient?: {
|
|
119
121
|
getDatasets: () => Promise<DatasetsResponse>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAGP,eAAe,EACf,eAAe,EACf,6BAA6B,EAE7B,yBAAyB,EAMzB,MAAM,EAEN,eAAe,EAMf,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,aAAa,EAEb,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,uBAAuB,
|
|
1
|
+
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAGP,eAAe,EACf,eAAe,EACf,6BAA6B,EAE7B,yBAAyB,EAMzB,MAAM,EAEN,eAAe,EAMf,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,aAAa,EAEb,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAE5B,KAAK,sBAAsB,EAC9B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACH,gBAAgB,EAChB,0BAA0B,EAC1B,8BAA8B,EAEjC,MAAM,UAAU,CAAC;AAOlB,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,iBAAiB;IACzB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,OAAO,YAAY;IACnB,UAAU,eAAe;IACzB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,KAAK,UAAU;CAClB;AAqFD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,GAAG,SAAS,KACzB,eAAe,GAAG,SASpB,CAAC;AA2BF,eAAO,MAAM,sBAAsB,UAAW;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,KAAG,iBAAiB,GAAG,IAuBvB,CAAC;AAEF,KAAK,WAAW,GAAG;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;CAC1B,CAAC;AAEF,KAAK,YAAY,GAAG,QAAQ,CACxB,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM,CAAC,CACpD,GACG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC;AAkDxD,KAAK,aAAa,GAAG;IACjB,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;CAClC,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,QAAQ,CAAC,IAAI,2BAA2B;IAExC,cAAc,IAAI,mBAAmB,CAEpC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAWhD;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGlC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAG7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGvC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAM1D;IAED,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,CAE3C;CACJ;AAID,KAAK,qBAAqB,GAAG;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAUF,eAAO,MAAM,wBAAwB,UAC1B,OAAO,KACf,qBAAqB,GAAG,SAe1B,CAAC;AAEF,qBAAa,uBAAwB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACvF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAM;IAExC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAK/C;IAGF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAGzC;IAEF,MAAM,EAAE,QAAQ,CAAC;IAEjB,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;IAE3C,YACI,WAAW,EAAE,yBAAyB,EACtC,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAC;QAAC,gBAAgB,CAAC,EAAE,OAAO,CAAA;KAAE,EAmCnE;IAED,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAE7D;IAED,OAAO,CAAC,oBAAoB;IAQ5B,OAAO,CAAC,8BAA8B;IAsBtC,OAAO,CAAC,4BAA4B;IAOpC;;;;;;;;;;OAUG;IACH,MAAM,CAAC,wBAAwB,CAC3B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CA0BpC;IAED,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,SAAS,oEAgBtE;YAEa,SAAS;YAoET,MAAM;IAWpB,MAAM,CAAC,iBAAiB,CACpB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,UAAU,EAAE,QAAQ,CAAC,cAAc,GAAG,SAAS,GAChD,uBAAuB,CAqBzB;YAEa,qBAAqB;YAUrB,aAAa;IAoCrB,WAAW,CACb,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC/B,GACF,OAAO,CAAC,IAAI,CAAC,CAuBf;IAED,OAAa,gBAAgB,CACzB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAShD;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,6BA0DN;IAEK,YAAY;;;;;;SAOjB;YAEa,sBAAsB;IAgE9B,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAmC3B;IAED,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,GAAE,MAAW,uBA+FzD;IAEK,aAAa,CAAC,EAChB,GAAG,EACH,MAAM,EACN,IAAI,EACJ,QAAQ,GACX,EAAE,8BAA8B,GAAG,OAAO,CAAC,sBAAsB,CAAC,CA2BlE;IAEK,iBAAiB,CACnB,EACI,GAAG,EACH,MAAM,EACN,IAAI,EACJ,QAAQ,EACR,kBAAkB,GACrB,EAAE,8BAA8B,EACjC,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GACV,OAAO,CAAC,0BAA0B,CAAC,CAoErC;YAEa,kBAAkB;IAWhC,OAAa,YAAY,CACrB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,cAAc,GAAE;QACZ,WAAW,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KASvD,GACH,OAAO,CAAC,eAAe,EAAE,CAAC,CA0B5B;IAED,OAAa,wBAAwB,CACjC,QAAQ,EAAE,eAAe,EAAE,EAC3B,YAAY,EAAE,MAAM,EACpB,qBAAqB,GAAE,CAAC,SAAS,EAAE,MAAM,KAAK;QAC1C,WAAW,EAAE,CACT,OAAO,EAAE,kBAAkB,KAC1B,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/B,KAAK,EAAE,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAgBpC,GACF,OAAO,CAAC,6BAA6B,CAAC,CAuFxC;IAED,OAAa,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA0DxE;CACJ"}
|
|
@@ -385,6 +385,7 @@ class BigqueryWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
385
385
|
}, {});
|
|
386
386
|
}
|
|
387
387
|
async createJob(query, options) {
|
|
388
|
+
const maximumBytesBilled = options.maximumBytesBilled ?? this.credentials.maximumBytesBilled;
|
|
388
389
|
const tags = options?.tags;
|
|
389
390
|
const labels = BigqueryWarehouseClient.sanitizeLabelsWithValues(tags);
|
|
390
391
|
const identityLabels = this.agentJobControls && tags
|
|
@@ -394,15 +395,18 @@ class BigqueryWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
394
395
|
query,
|
|
395
396
|
params: options?.values,
|
|
396
397
|
useLegacySql: false,
|
|
397
|
-
...(this.agentJobControls
|
|
398
|
+
...(this.agentJobControls || options.dryRun
|
|
399
|
+
? { useQueryCache: false }
|
|
400
|
+
: {}),
|
|
401
|
+
...(options.dryRun ? { dryRun: true } : {}),
|
|
398
402
|
// BigQuery has no session timezone; the `time_zone` connection
|
|
399
403
|
// property is the per-job equivalent — naive DATETIME coercions
|
|
400
404
|
// and offset-less literals are read in this zone.
|
|
401
405
|
connectionProperties: options?.timezone
|
|
402
406
|
? [{ key: 'time_zone', value: options.timezone }]
|
|
403
407
|
: undefined,
|
|
404
|
-
maximumBytesBilled:
|
|
405
|
-
? `${
|
|
408
|
+
maximumBytesBilled: maximumBytesBilled
|
|
409
|
+
? `${maximumBytesBilled}`
|
|
406
410
|
: undefined,
|
|
407
411
|
priority: this.credentials.priority,
|
|
408
412
|
jobTimeoutMs: this.credentials.timeoutSeconds &&
|
|
@@ -671,12 +675,39 @@ class BigqueryWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
671
675
|
console.error(`Unknown bigquery warehouse error reason: ${JSON.stringify(error, null, 2)}`);
|
|
672
676
|
return new common_1.WarehouseQueryError(`Bigquery warehouse error: ${error?.reason}${error?.message ? ` - ${error.message}` : ''}`);
|
|
673
677
|
}
|
|
674
|
-
async
|
|
678
|
+
async estimateQuery({ sql, values, tags, timezone, }) {
|
|
679
|
+
try {
|
|
680
|
+
const [job] = await this.createJob(sql, {
|
|
681
|
+
values,
|
|
682
|
+
tags,
|
|
683
|
+
timezone,
|
|
684
|
+
dryRun: true,
|
|
685
|
+
});
|
|
686
|
+
const totalBytesProcessed = Number(job.metadata?.statistics?.totalBytesProcessed ?? 0);
|
|
687
|
+
return {
|
|
688
|
+
totalBytesProcessed: Number.isFinite(totalBytesProcessed)
|
|
689
|
+
? totalBytesProcessed
|
|
690
|
+
: 0,
|
|
691
|
+
};
|
|
692
|
+
}
|
|
693
|
+
catch (e) {
|
|
694
|
+
this.throwIfGoogleOauthTokenError(e);
|
|
695
|
+
if (BigqueryWarehouseClient.isBigqueryError(e)) {
|
|
696
|
+
const responseError = e?.errors[0];
|
|
697
|
+
if (responseError) {
|
|
698
|
+
throw this.parseError(responseError, sql);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
throw e;
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
async executeAsyncQuery({ sql, values, tags, timezone, maximumBytesBilled, }, resultsStreamCallback) {
|
|
675
705
|
try {
|
|
676
706
|
const queryPhaseStart = performance.now();
|
|
677
707
|
const [job] = await this.createJob(sql, {
|
|
678
708
|
tags,
|
|
679
709
|
timezone,
|
|
710
|
+
maximumBytesBilled,
|
|
680
711
|
});
|
|
681
712
|
if (!job.id) {
|
|
682
713
|
throw new common_1.WarehouseQueryError('Missing BigQuery job ID. Please contact support.');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.586.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "2.
|
|
24
|
+
"@lightdash/common": "2.586.0",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|