@lightdash/warehouses 2.210.0 → 2.211.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.d.ts +2 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.js +3 -0
- package/dist/warehouseClients/DatabricksWarehouseClient.d.ts +15 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.js +158 -11
- package/dist/warehouseClients/DatabricksWarehouseClient.test.js +297 -18
- package/dist/warehouseClients/WarehouseBaseClient.d.ts +2 -1
- package/dist/warehouseClients/WarehouseBaseClient.d.ts.map +1 -1
- package/dist/warehouseClients/WarehouseBaseClient.js +3 -0
- package/dist/warehouseClients/WarehouseBaseSqlBuilder.d.ts +2 -1
- package/dist/warehouseClients/WarehouseBaseSqlBuilder.d.ts.map +1 -1
- package/dist/warehouseClients/WarehouseBaseSqlBuilder.js +4 -0
- package/package.json +2 -2
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { BigQuery, Dataset, DatasetsResponse, GetDatasetsOptions, QueryRowsResponse } from '@google-cloud/bigquery';
|
|
2
2
|
import bigquery from '@google-cloud/bigquery/build/src/types';
|
|
3
|
-
import { AnyType, BigqueryDataset, BigqueryProject, BigqueryProjectRecommendation, CreateBigqueryCredentials, Metric, PartitionColumn, SupportedDbtAdapter, TimeIntervalUnit, WarehouseQueryError, WarehouseResults, WarehouseTypes, type ResultNumericKind, type TimestampDomain } from '@lightdash/common';
|
|
3
|
+
import { AnyType, BigqueryDataset, BigqueryProject, BigqueryProjectRecommendation, CreateBigqueryCredentials, Metric, PartitionColumn, SupportedDbtAdapter, TimeIntervalUnit, UnnestSql, UnnestSqlArgs, WarehouseQueryError, WarehouseResults, WarehouseTypes, type ResultNumericKind, type TimestampDomain } from '@lightdash/common';
|
|
4
4
|
import { WarehouseCatalog, WarehouseExecuteAsyncQuery, WarehouseExecuteAsyncQueryArgs } from '../types';
|
|
5
5
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
6
6
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
@@ -52,6 +52,7 @@ export declare class BigquerySqlBuilder extends WarehouseBaseSqlBuilder {
|
|
|
52
52
|
getMedianSql(valueSql: string): string;
|
|
53
53
|
buildArray(elements: string[]): string;
|
|
54
54
|
buildArrayAgg(expression: string, orderBy?: string): string;
|
|
55
|
+
getUnnestSql(args: UnnestSqlArgs): UnnestSql;
|
|
55
56
|
}
|
|
56
57
|
export declare class BigqueryWarehouseClient extends WarehouseBaseClient<CreateBigqueryCredentials> {
|
|
57
58
|
private static readonly MAX_LABELS;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAGP,eAAe,EACf,eAAe,EACf,6BAA6B,EAE7B,yBAAyB,
|
|
1
|
+
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAGP,eAAe,EACf,eAAe,EACf,6BAA6B,EAE7B,yBAAyB,EAKzB,MAAM,EAEN,eAAe,EAMf,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,aAAa,EAEb,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,iBAAiB,EACtB,KAAK,eAAe,EAEvB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACH,gBAAgB,EAChB,0BAA0B,EAC1B,8BAA8B,EAEjC,MAAM,UAAU,CAAC;AAOlB,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,iBAAiB;IACzB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,OAAO,YAAY;IACnB,UAAU,eAAe;IACzB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,KAAK,UAAU;CAClB;AAqFD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,GAAG,SAAS,KACzB,eAAe,GAAG,SASpB,CAAC;AA2BF,eAAO,MAAM,sBAAsB,UAAW;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,KAAG,iBAAiB,GAAG,IAuBvB,CAAC;AAEF,KAAK,WAAW,GAAG;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;CAC1B,CAAC;AAEF,KAAK,YAAY,GAAG,QAAQ,CACxB,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM,CAAC,CACpD,GACG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC;AAkDxD,KAAK,aAAa,GAAG;IACjB,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;CAClC,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,QAAQ,CAAC,IAAI,2BAA2B;IAExC,cAAc,IAAI,mBAAmB,CAEpC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAWhD;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGlC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAG7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGvC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAM1D;IAED,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,CAE3C;CACJ;AAqCD,qBAAa,uBAAwB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACvF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAM;IAExC,MAAM,EAAE,QAAQ,CAAC;IAEjB,YAAY,WAAW,EAAE,yBAAyB,EAyBjD;IAED,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAE7D;IAED,OAAO,CAAC,oBAAoB;IAQ5B,OAAO,CAAC,8BAA8B;IAsBtC,OAAO,CAAC,4BAA4B;IAOpC;;;;;;;;;;OAUG;IACH,MAAM,CAAC,wBAAwB,CAC3B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAoBpC;IAED,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,SAAS,oEAgBtE;YAEa,SAAS;YA+BT,MAAM;YAWN,qBAAqB;YAUrB,aAAa;IAoCrB,WAAW,CACb,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC,CAuBf;IAED,OAAa,gBAAgB,CACzB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAShD;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,6BA0DN;IAEK,YAAY;;;;;;SAOjB;YAEa,sBAAsB;IAgE9B,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAmC3B;IAED,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,GAAE,MAAW,uBA+FzD;IAEK,iBAAiB,CACnB,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,8BAA8B,EACvD,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GACV,OAAO,CAAC,0BAA0B,CAAC,CA+DrC;YAEa,kBAAkB;IAWhC,OAAa,YAAY,CACrB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,cAAc,GAAE;QACZ,WAAW,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KASvD,GACH,OAAO,CAAC,eAAe,EAAE,CAAC,CA0B5B;IAED,OAAa,wBAAwB,CACjC,QAAQ,EAAE,eAAe,EAAE,EAC3B,YAAY,EAAE,MAAM,EACpB,qBAAqB,GAAE,CAAC,SAAS,EAAE,MAAM,KAAK;QAC1C,WAAW,EAAE,CACT,OAAO,EAAE,kBAAkB,KAC1B,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/B,KAAK,EAAE,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAgBpC,GACF,OAAO,CAAC,6BAA6B,CAAC,CAuFxC;IAED,OAAa,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA0DxE;CACJ"}
|
|
@@ -252,6 +252,9 @@ class BigquerySqlBuilder extends WarehouseBaseSqlBuilder_1.default {
|
|
|
252
252
|
}
|
|
253
253
|
return `ARRAY_AGG(${expression})`;
|
|
254
254
|
}
|
|
255
|
+
getUnnestSql(args) {
|
|
256
|
+
return (0, common_1.getBigqueryUnnestSql)(args);
|
|
257
|
+
}
|
|
255
258
|
}
|
|
256
259
|
exports.BigquerySqlBuilder = BigquerySqlBuilder;
|
|
257
260
|
const isRecord = (value) => typeof value === 'object' && value !== null;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ConnectionOptions } from '@databricks/sql/dist/contracts/IDBSQLClient';
|
|
2
2
|
import { TTypeId as DatabricksDataTypes } from '@databricks/sql/thrift/TCLIService_types';
|
|
3
|
-
import { AnyType, CreateDatabricksCredentials, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseResults, WarehouseTypes, type ResultNumericKind, type TimestampDomain } from '@lightdash/common';
|
|
3
|
+
import { AnyType, CreateDatabricksCredentials, Metric, SupportedDbtAdapter, TimeIntervalUnit, UnnestSql, UnnestSqlArgs, WarehouseResults, WarehouseTypes, type ResultNumericKind, type TimestampDomain } from '@lightdash/common';
|
|
4
4
|
import { WarehouseCatalog } from '../types';
|
|
5
5
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
6
6
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
@@ -33,7 +33,20 @@ export declare class DatabricksSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
|
33
33
|
getMedianSql(valueSql: string): string;
|
|
34
34
|
buildArray(elements: string[]): string;
|
|
35
35
|
buildArrayAgg(expression: string, orderBy?: string): string;
|
|
36
|
+
getUnnestSql(args: UnnestSqlArgs): UnnestSql;
|
|
36
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* The error conditions this client branches on, named exactly as in the
|
|
40
|
+
* Databricks catalogue (https://docs.databricks.com/error-messages/error-conditions).
|
|
41
|
+
* Only the ones we handle are listed; add here before matching a new one.
|
|
42
|
+
*/
|
|
43
|
+
export declare enum DatabricksErrorCondition {
|
|
44
|
+
/** SQLSTATE 42P01 */
|
|
45
|
+
TableOrViewNotFound = "TABLE_OR_VIEW_NOT_FOUND",
|
|
46
|
+
/** SQLSTATE 42601. What a runtime without `DESCRIBE ... AS JSON` answers. */
|
|
47
|
+
ParseSyntaxError = "PARSE_SYNTAX_ERROR"
|
|
48
|
+
}
|
|
49
|
+
export declare const getDatabricksErrorCondition: (error: unknown) => DatabricksErrorCondition | undefined;
|
|
37
50
|
export declare class DatabricksWarehouseClient extends WarehouseBaseClient<CreateDatabricksCredentials> {
|
|
38
51
|
schema: string;
|
|
39
52
|
catalog?: string;
|
|
@@ -47,6 +60,7 @@ export declare class DatabricksWarehouseClient extends WarehouseBaseClient<Creat
|
|
|
47
60
|
tags?: Record<string, string>;
|
|
48
61
|
timezone?: string;
|
|
49
62
|
}): Promise<void>;
|
|
63
|
+
private static fetchAllRows;
|
|
50
64
|
private static getTableColumns;
|
|
51
65
|
getCatalog(requests: {
|
|
52
66
|
database: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DatabricksWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DatabricksWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAqB,EACjB,iBAAiB,EACpB,MAAM,6CAA6C,CAAC;AAIrD,OAAO,EAAE,OAAO,IAAI,mBAAmB,EAAE,MAAM,0CAA0C,CAAC;AAC1F,OAAO,EACH,OAAO,EACP,2BAA2B,
|
|
1
|
+
{"version":3,"file":"DatabricksWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DatabricksWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAqB,EACjB,iBAAiB,EACpB,MAAM,6CAA6C,CAAC;AAIrD,OAAO,EAAE,OAAO,IAAI,mBAAmB,EAAE,MAAM,0CAA0C,CAAC;AAC1F,OAAO,EACH,OAAO,EACP,2BAA2B,EAM3B,MAAM,EAMN,mBAAmB,EACnB,gBAAgB,EAEhB,SAAS,EACT,aAAa,EAGb,gBAAgB,EAChB,cAAc,EACd,KAAK,iBAAiB,EACtB,KAAK,eAAe,EAEvB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAO5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE;;;;;;GAMG;AACH,eAAO,MAAM,kCAAkC,mBAAmB,CAAC;AAQnE,eAAO,MAAM,4BAA4B,aAC3B,MAAM,GAAG,SAAS,KAC7B,OAAsE,CAAC;AA8B1E,eAAO,MAAM,wBAAwB,UAAW;IAC5C,IAAI,EAAE,mBAAmB,CAAC;IAC1B,cAAc,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACrD,CAAC;CACL,KAAG,iBAAiB,GAAG,IAiBvB,CAAC;AAyHF,eAAO,MAAM,4BAA4B,SAC/B,MAAM,KACb,eAAe,GAAG,SASpB,CAAC;AAEF,qBAAa,oBAAqB,SAAQ,uBAAuB;IAC7D,QAAQ,CAAC,IAAI,6BAA6B;IAE1C,cAAc,IAAI,mBAAmB,CAEpC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAO1D;IAED,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,CAE3C;CACJ;AAgHD;;;;GAIG;AACH,oBAAY,wBAAwB;IAChC,qBAAqB;IACrB,mBAAmB,4BAA4B;IAC/C,6EAA6E;IAC7E,gBAAgB,uBAAuB;CAC1C;AAKD,eAAO,MAAM,2BAA2B,UAC7B,OAAO,KACf,wBAAwB,GAAG,SAO7B,CAAC;AAgBF,qBAAa,yBAA0B,SAAQ,mBAAmB,CAAC,2BAA2B,CAAC;IAC3F,MAAM,EAAE,MAAM,CAAC;IAEf,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,iBAAiB,EAAE,iBAAiB,CAAC;IAErC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IAEzC,YAAY,WAAW,EAAE,2BAA2B,EA+CnD;YAEa,WAAW;YA6CX,oBAAoB;IA0E5B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC,CAkCf;mBAEoB,YAAY;mBAsBZ,eAAe;IA8D9B,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,6BAkGN;IAIK,YAAY;;;;;SAcjB;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAyB3B;CACJ;AAED;;GAEG;AACH,eAAO,MAAM,kCAAkC,SACrC,MAAM,YACF,MAAM,gBACF,MAAM,KACrB,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,CA+BxD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,2BAA2B,SAC9B,MAAM,YACF,MAAM,gBACF,MAAM,iBACL,MAAM,KACtB,OAAO,CAAC;IACP,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACrB,CAqCA,CAAC"}
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.refreshDatabricksOAuthToken = exports.exchangeDatabricksOAuthCredentials = exports.DatabricksWarehouseClient = exports.DatabricksSqlBuilder = exports.getDatabricksTimestampDomain = exports.getDatabricksNumericKind = exports.isDatabricksCliOAuthClientId = exports.DATABRICKS_DEFAULT_OAUTH_CLIENT_ID = void 0;
|
|
6
|
+
exports.refreshDatabricksOAuthToken = exports.exchangeDatabricksOAuthCredentials = exports.DatabricksWarehouseClient = exports.getDatabricksErrorCondition = exports.DatabricksErrorCondition = exports.DatabricksSqlBuilder = exports.getDatabricksTimestampDomain = exports.getDatabricksNumericKind = exports.isDatabricksCliOAuthClientId = exports.DATABRICKS_DEFAULT_OAUTH_CLIENT_ID = void 0;
|
|
7
7
|
const sql_1 = require("@databricks/sql");
|
|
8
8
|
const StatusError_1 = __importDefault(require("@databricks/sql/dist/errors/StatusError"));
|
|
9
9
|
const TCLIService_types_1 = require("@databricks/sql/thrift/TCLIService_types");
|
|
@@ -231,6 +231,9 @@ class DatabricksSqlBuilder extends WarehouseBaseSqlBuilder_1.default {
|
|
|
231
231
|
}
|
|
232
232
|
return `COLLECT_LIST(${expression})`;
|
|
233
233
|
}
|
|
234
|
+
getUnnestSql(args) {
|
|
235
|
+
return (0, common_1.getDatabricksUnnestSql)(args);
|
|
236
|
+
}
|
|
234
237
|
}
|
|
235
238
|
exports.DatabricksSqlBuilder = DatabricksSqlBuilder;
|
|
236
239
|
const DATABRICKS_SOCKET_TIMEOUT_MS = 60000;
|
|
@@ -238,9 +241,99 @@ const DATABRICKS_QUERY_TIMEOUT_SECONDS = 300;
|
|
|
238
241
|
// @databricks/sql defaults fetchChunk to 100k rows, which materializes entire
|
|
239
242
|
// wide results in one array and can OOM the worker on large queries
|
|
240
243
|
const DATABRICKS_FETCH_CHUNK_MAX_ROWS = 5000;
|
|
244
|
+
const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
245
|
+
// Elements with no path to their contents. A struct element exposes its
|
|
246
|
+
// fields and a scalar element is the value; these expose nothing, so the
|
|
247
|
+
// column stays an opaque scalar rather than posing as an array of scalars.
|
|
248
|
+
const OPAQUE_ELEMENT_TYPES = new Set(['array', 'map', 'variant']);
|
|
249
|
+
// A plain TIMESTAMP column is described as timestamp_ltz.
|
|
250
|
+
const scalarTypeName = (name) => name === 'timestamp_ltz' ? DatabricksTypes.TIMESTAMP : name.toUpperCase();
|
|
251
|
+
const describeNode = (type) => {
|
|
252
|
+
const name = typeof type.name === 'string' ? type.name : 'string';
|
|
253
|
+
if (name === 'struct') {
|
|
254
|
+
return {
|
|
255
|
+
typeName: DatabricksTypes.STRUCT,
|
|
256
|
+
shape: { repeated: false, record: true },
|
|
257
|
+
fields: type.fields,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
if (name === 'array') {
|
|
261
|
+
const element = isRecord(type.element_type) ? type.element_type : {};
|
|
262
|
+
const elementName = typeof element.name === 'string' ? element.name : 'string';
|
|
263
|
+
if (elementName === 'struct') {
|
|
264
|
+
return {
|
|
265
|
+
typeName: DatabricksTypes.STRUCT,
|
|
266
|
+
shape: { repeated: true, record: true },
|
|
267
|
+
fields: element.fields,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
if (OPAQUE_ELEMENT_TYPES.has(elementName)) {
|
|
271
|
+
return {
|
|
272
|
+
typeName: DatabricksTypes.ARRAY,
|
|
273
|
+
shape: undefined,
|
|
274
|
+
fields: undefined,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
// The element type types the array's value dimension, as BigQuery's
|
|
278
|
+
// REPEATED scalar does.
|
|
279
|
+
return {
|
|
280
|
+
typeName: scalarTypeName(elementName),
|
|
281
|
+
shape: { repeated: true, record: false },
|
|
282
|
+
fields: undefined,
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
return {
|
|
286
|
+
typeName: scalarTypeName(name),
|
|
287
|
+
shape: undefined,
|
|
288
|
+
fields: undefined,
|
|
289
|
+
};
|
|
290
|
+
};
|
|
291
|
+
/**
|
|
292
|
+
* Walks the `columns` of a `DESCRIBE TABLE ... AS JSON` result depth-first,
|
|
293
|
+
* emitting every node under its dotted path. A struct is `{name: "struct",
|
|
294
|
+
* fields}`, an array `{name: "array", element_type}`; an array of structs is
|
|
295
|
+
* both, and its children are the element's fields.
|
|
296
|
+
*/
|
|
297
|
+
const flattenDescribedFields = (fields, prefix = '') => {
|
|
298
|
+
if (!Array.isArray(fields))
|
|
299
|
+
return [];
|
|
300
|
+
return fields.flatMap((field) => {
|
|
301
|
+
if (!isRecord(field) ||
|
|
302
|
+
typeof field.name !== 'string' ||
|
|
303
|
+
!isRecord(field.type)) {
|
|
304
|
+
return [];
|
|
305
|
+
}
|
|
306
|
+
const path = prefix ? `${prefix}.${field.name}` : field.name;
|
|
307
|
+
const { typeName, shape, fields: children } = describeNode(field.type);
|
|
308
|
+
return [
|
|
309
|
+
{ path, typeName, shape },
|
|
310
|
+
...flattenDescribedFields(children, path),
|
|
311
|
+
];
|
|
312
|
+
});
|
|
313
|
+
};
|
|
241
314
|
const getDatabricksErrorMessage = (error) => error instanceof StatusError_1.default && error.message
|
|
242
315
|
? error.message
|
|
243
316
|
: (0, common_1.getErrorMessage)(error);
|
|
317
|
+
/**
|
|
318
|
+
* The error conditions this client branches on, named exactly as in the
|
|
319
|
+
* Databricks catalogue (https://docs.databricks.com/error-messages/error-conditions).
|
|
320
|
+
* Only the ones we handle are listed; add here before matching a new one.
|
|
321
|
+
*/
|
|
322
|
+
var DatabricksErrorCondition;
|
|
323
|
+
(function (DatabricksErrorCondition) {
|
|
324
|
+
/** SQLSTATE 42P01 */
|
|
325
|
+
DatabricksErrorCondition["TableOrViewNotFound"] = "TABLE_OR_VIEW_NOT_FOUND";
|
|
326
|
+
/** SQLSTATE 42601. What a runtime without `DESCRIBE ... AS JSON` answers. */
|
|
327
|
+
DatabricksErrorCondition["ParseSyntaxError"] = "PARSE_SYNTAX_ERROR";
|
|
328
|
+
})(DatabricksErrorCondition || (exports.DatabricksErrorCondition = DatabricksErrorCondition = {}));
|
|
329
|
+
// Databricks prefixes every SQL error with its condition, `[CONDITION] message
|
|
330
|
+
// ... SQLSTATE: xxxxx`. The Node driver drops the Thrift sqlState, so the
|
|
331
|
+
// prefix is the only structured signal that reaches us.
|
|
332
|
+
const getDatabricksErrorCondition = (error) => {
|
|
333
|
+
const condition = /^\[([A-Z_.]+)\]/.exec(getDatabricksErrorMessage(error))?.[1];
|
|
334
|
+
return Object.values(DatabricksErrorCondition).find((known) => known === condition);
|
|
335
|
+
};
|
|
336
|
+
exports.getDatabricksErrorCondition = getDatabricksErrorCondition;
|
|
244
337
|
// Don't let close errors override the original error
|
|
245
338
|
const closeQuietly = async (close, context) => {
|
|
246
339
|
try {
|
|
@@ -418,12 +511,8 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
418
511
|
}
|
|
419
512
|
/* eslint-enable no-await-in-loop */
|
|
420
513
|
}
|
|
421
|
-
static async
|
|
422
|
-
const query = await
|
|
423
|
-
catalogName: request.database,
|
|
424
|
-
schemaName: request.schema,
|
|
425
|
-
tableName: request.table,
|
|
426
|
-
});
|
|
514
|
+
static async fetchAllRows(operation, label) {
|
|
515
|
+
const query = await operation;
|
|
427
516
|
try {
|
|
428
517
|
return (await query.fetchAll());
|
|
429
518
|
}
|
|
@@ -432,10 +521,63 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
432
521
|
await query.close();
|
|
433
522
|
}
|
|
434
523
|
catch (e) {
|
|
435
|
-
console.error(
|
|
524
|
+
console.error(`Error closing Databricks query on ${label}`, e);
|
|
436
525
|
}
|
|
437
526
|
}
|
|
438
527
|
}
|
|
528
|
+
// DESCRIBE ... AS JSON gives the nested tree; older all-purpose clusters
|
|
529
|
+
// reject the JSON form, so those tables keep the flat column list and
|
|
530
|
+
// carry the reason for the compiler to surface.
|
|
531
|
+
// DESCRIBE ... AS JSON gives the nested tree. A runtime that does not know
|
|
532
|
+
// the JSON form rejects the statement itself; only that case falls back
|
|
533
|
+
// to the flat column list, any other failure fails the fetch as before.
|
|
534
|
+
static async getTableColumns(session, request) {
|
|
535
|
+
const q = '`';
|
|
536
|
+
const tableRef = `${q}${request.database}${q}.${q}${request.schema}${q}.${q}${request.table}${q}`;
|
|
537
|
+
let rows;
|
|
538
|
+
try {
|
|
539
|
+
rows = await DatabricksWarehouseClient.fetchAllRows(session.executeStatement(`DESCRIBE TABLE EXTENDED ${tableRef} AS JSON`), 'getCatalog');
|
|
540
|
+
}
|
|
541
|
+
catch (e) {
|
|
542
|
+
const condition = (0, exports.getDatabricksErrorCondition)(e);
|
|
543
|
+
if (condition === DatabricksErrorCondition.TableOrViewNotFound) {
|
|
544
|
+
return { discovery: 'described', columns: [] };
|
|
545
|
+
}
|
|
546
|
+
if ((0, DatabricksWarehouseStartupRetry_1.isDatabricksWarehouseStartingError)(e) ||
|
|
547
|
+
condition !== DatabricksErrorCondition.ParseSyntaxError) {
|
|
548
|
+
throw e;
|
|
549
|
+
}
|
|
550
|
+
const message = getDatabricksErrorMessage(e);
|
|
551
|
+
const flatColumns = (await DatabricksWarehouseClient.fetchAllRows(session.getColumns({
|
|
552
|
+
catalogName: request.database,
|
|
553
|
+
schemaName: request.schema,
|
|
554
|
+
tableName: request.table,
|
|
555
|
+
}), 'getCatalog'));
|
|
556
|
+
return {
|
|
557
|
+
discovery: 'unsupported',
|
|
558
|
+
columns: flatColumns.map((column) => ({
|
|
559
|
+
path: column.COLUMN_NAME,
|
|
560
|
+
typeName: column.TYPE_NAME,
|
|
561
|
+
shape: undefined,
|
|
562
|
+
})),
|
|
563
|
+
reason: `Databricks did not describe ${request.table} as JSON (${message}); nested columns need a SQL warehouse or Databricks Runtime 16.2 or newer.`,
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
const json = rows[0] ? Object.values(rows[0])[0] : undefined;
|
|
567
|
+
try {
|
|
568
|
+
const description = typeof json === 'string' ? JSON.parse(json) : undefined;
|
|
569
|
+
if (!isRecord(description) || !Array.isArray(description.columns)) {
|
|
570
|
+
throw new Error('no columns in the JSON description');
|
|
571
|
+
}
|
|
572
|
+
return {
|
|
573
|
+
discovery: 'described',
|
|
574
|
+
columns: flattenDescribedFields(description.columns),
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
catch (e) {
|
|
578
|
+
throw new common_1.WarehouseQueryError(`Could not read the description of ${tableRef}: ${(0, common_1.getErrorMessage)(e)}`);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
439
581
|
async getCatalog(requests) {
|
|
440
582
|
const retry = new DatabricksWarehouseStartupRetry_1.DatabricksWarehouseStartupRetry();
|
|
441
583
|
const results = new Map();
|
|
@@ -477,10 +619,15 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
477
619
|
return requests.reduce((acc, { schema, table }, index) => {
|
|
478
620
|
acc[catalog][schema] = acc[catalog][schema] || {};
|
|
479
621
|
acc[catalog][schema][table] = {};
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
622
|
+
const described = results.get(index);
|
|
623
|
+
described?.columns.forEach(({ path, typeName, shape }) => {
|
|
624
|
+
acc[catalog][schema][table][path] = mapFieldType(typeName);
|
|
625
|
+
(0, common_1.setCatalogTimestampDomain)(acc, catalog, schema, table, path, (0, exports.getDatabricksTimestampDomain)(typeName));
|
|
626
|
+
(0, common_1.setCatalogNestedColumnShape)(acc, catalog, schema, table, path, shape);
|
|
483
627
|
});
|
|
628
|
+
if (described?.discovery === 'unsupported') {
|
|
629
|
+
(0, common_1.setCatalogNestedColumnsUnavailable)(acc, catalog, schema, table, described.reason);
|
|
630
|
+
}
|
|
484
631
|
return acc;
|
|
485
632
|
}, { [catalog]: {} });
|
|
486
633
|
}
|