@lightdash/warehouses 0.3439.0 → 0.3441.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/AthenaWarehouseClient.d.ts +2 -1
- package/dist/warehouseClients/AthenaWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/AthenaWarehouseClient.js +27 -13
- package/dist/warehouseClients/BigqueryWarehouseClient.d.ts +2 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.js +20 -7
- package/dist/warehouseClients/BigqueryWarehouseClient.test.js +1 -1
- package/dist/warehouseClients/ClickhouseWarehouseClient.d.ts +2 -1
- package/dist/warehouseClients/ClickhouseWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/ClickhouseWarehouseClient.js +21 -5
- package/dist/warehouseClients/DatabricksWarehouseClient.d.ts +2 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.js +23 -8
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +2 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +17 -5
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +5 -0
- package/dist/warehouseClients/PostgresWarehouseClient.d.ts +5 -5
- package/dist/warehouseClients/PostgresWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/PostgresWarehouseClient.js +19 -2
- package/dist/warehouseClients/PostgresWarehouseClient.test.js +1 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.d.ts +2 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.js +21 -3
- package/dist/warehouseClients/SnowflakeWarehouseClient.mock.d.ts +9 -0
- package/dist/warehouseClients/SnowflakeWarehouseClient.mock.d.ts.map +1 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.mock.js +6 -0
- package/dist/warehouseClients/TrinoWarehouseClient.d.ts +2 -1
- package/dist/warehouseClients/TrinoWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/TrinoWarehouseClient.js +16 -4
- package/dist/warehouseClients/TrinoWarehouseClient.test.js +1 -1
- package/dist/warehouseClients/WarehouseBaseClient.d.ts +2 -2
- package/dist/warehouseClients/WarehouseBaseClient.d.ts.map +1 -1
- package/dist/warehouseClients/WarehouseBaseClient.js +5 -2
- package/dist/warehouseClients/WarehouseClient.mock.d.ts +2 -0
- package/dist/warehouseClients/WarehouseClient.mock.d.ts.map +1 -1
- package/dist/warehouseClients/WarehouseClient.mock.js +8 -1
- package/dist/warehouseClients/timestampDomain.test.d.ts +2 -0
- package/dist/warehouseClients/timestampDomain.test.d.ts.map +1 -0
- package/dist/warehouseClients/timestampDomain.test.js +101 -0
- package/package.json +2 -2
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AthenaClient } from '@aws-sdk/client-athena';
|
|
2
|
-
import { CreateAthenaCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseResults, WarehouseTypes } from '@lightdash/common';
|
|
2
|
+
import { CreateAthenaCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseResults, WarehouseTypes, type TimestampDomain } from '@lightdash/common';
|
|
3
3
|
import { WarehouseCatalog } from '../types';
|
|
4
4
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
5
5
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
@@ -28,6 +28,7 @@ export declare enum AthenaTypes {
|
|
|
28
28
|
IPADDRESS = "ipaddress",
|
|
29
29
|
UUID = "uuid"
|
|
30
30
|
}
|
|
31
|
+
export declare const getAthenaTimestampDomain: (type: AthenaTypes | string) => TimestampDomain | undefined;
|
|
31
32
|
export declare const convertDataTypeToDimensionType: (type: AthenaTypes | string) => DimensionType;
|
|
32
33
|
export declare class AthenaSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
33
34
|
readonly type = WarehouseTypes.ATHENA;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AthenaWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/AthenaWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,YAAY,EAOf,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAGH,uBAAuB,EACvB,aAAa,EAEb,MAAM,
|
|
1
|
+
{"version":3,"file":"AthenaWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/AthenaWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,YAAY,EAOf,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAGH,uBAAuB,EACvB,aAAa,EAEb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAGhB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE,oBAAY,WAAW;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,wBAAwB;IAC/B,SAAS,cAAc;IACvB,YAAY,6BAA6B;IACzC,KAAK,UAAU;IACf,GAAG,QAAQ;IACX,GAAG,QAAQ;IACX,SAAS,cAAc;IACvB,IAAI,SAAS;CAChB;AAGD,eAAO,MAAM,wBAAwB,GACjC,MAAM,WAAW,GAAG,MAAM,KAC3B,eAAe,GAAG,SASpB,CAAC;AAEF,eAAO,MAAM,8BAA8B,GACvC,MAAM,WAAW,GAAG,MAAM,KAC3B,aAwBF,CAAC;AAiGF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,QAAQ,CAAC,IAAI,yBAAyB;IAEtC,cAAc,IAAI,mBAAmB;IAKrC,0BAA0B,IAAI,OAAO;IAIrC,wBAAwB,IAAI,MAAM;IAIlC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAajD,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmBnC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM;IAM7D,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM;IAKT,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;CAIzC;AAKD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,uBAAuB,CAAC;IACnF,MAAM,EAAE,YAAY,CAAC;gBAET,WAAW,EAAE,uBAAuB;YAyDlC,sBAAsB;IAiDpC,OAAO,CAAC,UAAU;IA+BZ,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA0GV,UAAU,CACZ,QAAQ,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAChE,OAAO,CAAC,gBAAgB,CAAC;IA0DtB,YAAY,IAAI,OAAO,CACzB;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CACxD;IAwCK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,CAAC;IAyC5B,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAGlC"}
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.AthenaWarehouseClient = exports.AthenaSqlBuilder = exports.convertDataTypeToDimensionType = exports.AthenaTypes = void 0;
|
|
6
|
+
exports.AthenaWarehouseClient = exports.AthenaSqlBuilder = exports.convertDataTypeToDimensionType = exports.getAthenaTimestampDomain = exports.AthenaTypes = void 0;
|
|
7
7
|
const client_athena_1 = require("@aws-sdk/client-athena");
|
|
8
8
|
const credential_providers_1 = require("@aws-sdk/credential-providers");
|
|
9
9
|
const common_1 = require("@lightdash/common");
|
|
@@ -37,6 +37,18 @@ var AthenaTypes;
|
|
|
37
37
|
AthenaTypes["IPADDRESS"] = "ipaddress";
|
|
38
38
|
AthenaTypes["UUID"] = "uuid";
|
|
39
39
|
})(AthenaTypes || (exports.AthenaTypes = AthenaTypes = {}));
|
|
40
|
+
// Athena follows Trino semantics: bare timestamp is naive, with time zone is aware
|
|
41
|
+
const getAthenaTimestampDomain = (type) => {
|
|
42
|
+
switch (type.toLowerCase().replace(/\(\d+(,\s*\d+)?\)/, '')) {
|
|
43
|
+
case AthenaTypes.TIMESTAMP:
|
|
44
|
+
return 'naive';
|
|
45
|
+
case AthenaTypes.TIMESTAMP_TZ:
|
|
46
|
+
return 'aware';
|
|
47
|
+
default:
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
exports.getAthenaTimestampDomain = getAthenaTimestampDomain;
|
|
40
52
|
const convertDataTypeToDimensionType = (type) => {
|
|
41
53
|
const normalizedType = type.toLowerCase().replace(/\(\d+(,\s*\d+)?\)/, '');
|
|
42
54
|
switch (normalizedType) {
|
|
@@ -405,10 +417,12 @@ class AthenaWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
405
417
|
const { database, schema, table, columns } = result;
|
|
406
418
|
acc[database] = acc[database] || {};
|
|
407
419
|
acc[database][schema] = acc[database][schema] || {};
|
|
408
|
-
acc[database][schema][table] =
|
|
409
|
-
|
|
410
|
-
[normalizeColumnName(name)]
|
|
411
|
-
|
|
420
|
+
acc[database][schema][table] = {};
|
|
421
|
+
columns.forEach(({ name, type }) => {
|
|
422
|
+
acc[database][schema][table][normalizeColumnName(name)] =
|
|
423
|
+
(0, exports.convertDataTypeToDimensionType)(type);
|
|
424
|
+
(0, common_1.setCatalogTimestampDomain)(acc, database, schema, table, normalizeColumnName(name), (0, exports.getAthenaTimestampDomain)(type));
|
|
425
|
+
});
|
|
412
426
|
}
|
|
413
427
|
return acc;
|
|
414
428
|
}, {});
|
|
@@ -456,15 +470,15 @@ class AthenaWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
456
470
|
}));
|
|
457
471
|
const columns = response.TableMetadata?.Columns || [];
|
|
458
472
|
const result = {
|
|
459
|
-
[db]: {
|
|
460
|
-
[sch]: {
|
|
461
|
-
[tableName]: columns.reduce((acc, col) => ({
|
|
462
|
-
...acc,
|
|
463
|
-
[normalizeColumnName(col.Name || '')]: (0, exports.convertDataTypeToDimensionType)(col.Type || 'varchar'),
|
|
464
|
-
}), {}),
|
|
465
|
-
},
|
|
466
|
-
},
|
|
473
|
+
[db]: { [sch]: { [tableName]: {} } },
|
|
467
474
|
};
|
|
475
|
+
columns.forEach((col) => {
|
|
476
|
+
const columnName = normalizeColumnName(col.Name || '');
|
|
477
|
+
const rawType = col.Type || 'varchar';
|
|
478
|
+
result[db][sch][tableName][columnName] =
|
|
479
|
+
(0, exports.convertDataTypeToDimensionType)(rawType);
|
|
480
|
+
(0, common_1.setCatalogTimestampDomain)(result, db, sch, tableName, columnName, (0, exports.getAthenaTimestampDomain)(rawType));
|
|
481
|
+
});
|
|
468
482
|
return result;
|
|
469
483
|
}
|
|
470
484
|
catch (e) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { BigQuery, Dataset, DatasetsResponse, GetDatasetsOptions, QueryRowsResponse } from '@google-cloud/bigquery';
|
|
2
2
|
import bigquery from '@google-cloud/bigquery/build/src/types';
|
|
3
|
-
import { AnyType, BigqueryDataset, BigqueryProject, BigqueryProjectRecommendation, CreateBigqueryCredentials, DimensionType, Metric, PartitionColumn, SupportedDbtAdapter, TimeIntervalUnit, WarehouseQueryError, WarehouseResults, WarehouseTypes } from '@lightdash/common';
|
|
3
|
+
import { AnyType, BigqueryDataset, BigqueryProject, BigqueryProjectRecommendation, CreateBigqueryCredentials, DimensionType, Metric, PartitionColumn, SupportedDbtAdapter, TimeIntervalUnit, WarehouseQueryError, WarehouseResults, WarehouseTypes, type TimestampDomain } from '@lightdash/common';
|
|
4
4
|
import { WarehouseCatalog, WarehouseExecuteAsyncQuery, WarehouseExecuteAsyncQueryArgs } from '../types';
|
|
5
5
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
6
6
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
@@ -24,6 +24,7 @@ export declare enum BigqueryFieldType {
|
|
|
24
24
|
STRUCT = "STRUCT",
|
|
25
25
|
ARRAY = "ARRAY"
|
|
26
26
|
}
|
|
27
|
+
export declare const getBigqueryTimestampDomain: (type: string | undefined) => TimestampDomain | undefined;
|
|
27
28
|
type TableSchema = {
|
|
28
29
|
fields: SchemaFields[];
|
|
29
30
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAEP,eAAe,EACf,eAAe,EACf,6BAA6B,EAC7B,yBAAyB,EACzB,aAAa,EAEb,MAAM,EAEN,eAAe,
|
|
1
|
+
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAEP,eAAe,EACf,eAAe,EACf,6BAA6B,EAC7B,yBAAyB,EACzB,aAAa,EAEb,MAAM,EAEN,eAAe,EAKf,mBAAmB,EACnB,gBAAgB,EAEhB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACH,gBAAgB,EAChB,0BAA0B,EAC1B,8BAA8B,EAEjC,MAAM,UAAU,CAAC;AAMlB,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,iBAAiB;IACzB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,OAAO,YAAY;IACnB,UAAU,eAAe;IACzB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,KAAK,UAAU;CAClB;AA8BD,eAAO,MAAM,0BAA0B,GACnC,MAAM,MAAM,GAAG,SAAS,KACzB,eAAe,GAAG,SASpB,CAAC;AA0BF,KAAK,WAAW,GAAG;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;CAC1B,CAAC;AAEF,KAAK,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;AAehF,KAAK,aAAa,GAAG;IACjB,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;CAClC,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,QAAQ,CAAC,IAAI,2BAA2B;IAExC,cAAc,IAAI,mBAAmB;IAIrC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAajD,iBAAiB,IAAI,MAAM;IAI3B,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmBnC,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM;IAKnC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM;IAM7D,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM;IAKT,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAKtC,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAKtC,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM;CAO9D;AAED,qBAAa,uBAAwB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACvF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAM;IAExC,MAAM,EAAE,QAAQ,CAAC;gBAEL,WAAW,EAAE,yBAAyB;IA2BlD,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa;IAI9D;;;;;;;;;;OAUG;IACH,MAAM,CAAC,wBAAwB,CAC3B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS;IAgCrC,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,SAAS;;;YAczD,SAAS;YA+BT,MAAM;YAWN,qBAAqB;YAUrB,aAAa;IAoCrB,WAAW,CACb,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;WAwBH,gBAAgB,CACzB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAW3C,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE;IAkDD,YAAY;;;;;;IA+DZ,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,CAAC;IAqB5B,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,GAAE,MAAW;IAiGpD,iBAAiB,CACnB,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,8BAA8B,EACvD,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GACV,OAAO,CAAC,0BAA0B,CAAC;YAgExB,kBAAkB;WAWnB,YAAY,CACrB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,cAAc,GAAE;QACZ,WAAW,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KASvD,GACH,OAAO,CAAC,eAAe,EAAE,CAAC;WA4BhB,wBAAwB,CACjC,QAAQ,EAAE,eAAe,EAAE,EAC3B,YAAY,EAAE,MAAM,EACpB,qBAAqB,GAAE,CAAC,SAAS,EAAE,MAAM,KAAK;QAC1C,WAAW,EAAE,CACT,OAAO,EAAE,kBAAkB,KAC1B,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/B,KAAK,EAAE,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAgBpC,GACF,OAAO,CAAC,6BAA6B,CAAC;WAyF5B,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CA2D5E"}
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.BigqueryWarehouseClient = exports.BigquerySqlBuilder = exports.BigqueryFieldType = void 0;
|
|
6
|
+
exports.BigqueryWarehouseClient = exports.BigquerySqlBuilder = exports.getBigqueryTimestampDomain = exports.BigqueryFieldType = void 0;
|
|
7
7
|
const bigquery_1 = require("@google-cloud/bigquery");
|
|
8
8
|
const common_1 = require("@lightdash/common");
|
|
9
9
|
const big_js_1 = __importDefault(require("big.js"));
|
|
@@ -55,6 +55,17 @@ const parseCell = (cell) => {
|
|
|
55
55
|
}
|
|
56
56
|
return `${cell}`;
|
|
57
57
|
};
|
|
58
|
+
const getBigqueryTimestampDomain = (type) => {
|
|
59
|
+
switch (type) {
|
|
60
|
+
case BigqueryFieldType.DATETIME:
|
|
61
|
+
return 'naive';
|
|
62
|
+
case BigqueryFieldType.TIMESTAMP:
|
|
63
|
+
return 'aware';
|
|
64
|
+
default:
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
exports.getBigqueryTimestampDomain = getBigqueryTimestampDomain;
|
|
58
69
|
const mapFieldType = (type) => {
|
|
59
70
|
switch (type) {
|
|
60
71
|
case BigqueryFieldType.DATE:
|
|
@@ -336,11 +347,13 @@ class BigqueryWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
336
347
|
const [database, schema, table, tableSchema] = result;
|
|
337
348
|
acc[database] = acc[database] || {};
|
|
338
349
|
acc[database][schema] = acc[database][schema] || {};
|
|
339
|
-
acc[database][schema][table] =
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
350
|
+
acc[database][schema][table] = {};
|
|
351
|
+
tableSchema.fields.forEach(({ name, type }) => {
|
|
352
|
+
if (name === undefined)
|
|
353
|
+
return;
|
|
354
|
+
acc[database][schema][table][name] = mapFieldType(type);
|
|
355
|
+
(0, common_1.setCatalogTimestampDomain)(acc, database, schema, table, name, (0, exports.getBigqueryTimestampDomain)(type));
|
|
356
|
+
});
|
|
344
357
|
}
|
|
345
358
|
return acc;
|
|
346
359
|
}, {});
|
|
@@ -399,7 +412,7 @@ class BigqueryWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
399
412
|
table_name: schemas[2],
|
|
400
413
|
column_name: column.name,
|
|
401
414
|
data_type: column.type,
|
|
402
|
-
})), mapFieldType);
|
|
415
|
+
})), mapFieldType, exports.getBigqueryTimestampDomain);
|
|
403
416
|
}
|
|
404
417
|
parseError(error, query = '') {
|
|
405
418
|
// if the error has no reason, return a generic error
|
|
@@ -37,7 +37,7 @@ describe('BigqueryWarehouseClient', () => {
|
|
|
37
37
|
.mockImplementationOnce(() => BigqueryWarehouseClient_mock_1.getTableResponse);
|
|
38
38
|
bigquery_1.Dataset.prototype.table = getTableMock;
|
|
39
39
|
const warehouse = new BigqueryWarehouseClient_1.BigqueryWarehouseClient(BigqueryWarehouseClient_mock_1.credentials);
|
|
40
|
-
expect(await warehouse.getCatalog(WarehouseClient_mock_1.config)).toEqual(WarehouseClient_mock_1.
|
|
40
|
+
expect(await warehouse.getCatalog(WarehouseClient_mock_1.config)).toEqual(WarehouseClient_mock_1.expectedWarehouseSchemaWithAwareTimestamp);
|
|
41
41
|
expect(getTableMock).toHaveBeenCalledTimes(1);
|
|
42
42
|
expect(BigqueryWarehouseClient_mock_1.getTableResponse.getMetadata).toHaveBeenCalledTimes(1);
|
|
43
43
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type ClickHouseClient } from '@clickhouse/client';
|
|
2
|
-
import { AnyType, CreateClickhouseCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseResults, WarehouseTypes } from '@lightdash/common';
|
|
2
|
+
import { AnyType, CreateClickhouseCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseResults, WarehouseTypes, type TimestampDomain } from '@lightdash/common';
|
|
3
3
|
import { WarehouseCatalog } from '../types';
|
|
4
4
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
5
5
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
@@ -42,6 +42,7 @@ interface TableInfo {
|
|
|
42
42
|
schema?: string;
|
|
43
43
|
table: string;
|
|
44
44
|
}
|
|
45
|
+
export declare const getClickhouseTimestampDomain: (type: ClickhouseTypes | string) => TimestampDomain | undefined;
|
|
45
46
|
export declare const convertDataTypeToDimensionType: (type: ClickhouseTypes | string) => DimensionType;
|
|
46
47
|
export declare class ClickhouseSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
47
48
|
readonly type = WarehouseTypes.CLICKHOUSE;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ClickhouseWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/ClickhouseWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,KAAK,gBAAgB,EAExB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACH,OAAO,EACP,2BAA2B,EAC3B,aAAa,EAEb,MAAM,
|
|
1
|
+
{"version":3,"file":"ClickhouseWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/ClickhouseWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,KAAK,gBAAgB,EAExB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACH,OAAO,EACP,2BAA2B,EAC3B,aAAa,EAEb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAGhB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE,oBAAY,eAAe;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,SAAS,cAAc;IACvB,UAAU,eAAe;IACzB,UAAU,eAAe;IACzB,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,WAAW,gBAAgB;IAC3B,IAAI,SAAS;IACb,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,UAAU,eAAe;IACzB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,GAAG,QAAQ;IACX,QAAQ,aAAa;IACrB,cAAc,mBAAmB;IACjC,IAAI,SAAS;IACb,IAAI,SAAS;CAChB;AAED,UAAU,SAAS;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACjB;AAmBD,eAAO,MAAM,4BAA4B,GACrC,MAAM,eAAe,GAAG,MAAM,KAC/B,eAAe,GAAG,SAQpB,CAAC;AAEF,eAAO,MAAM,8BAA8B,GACvC,MAAM,eAAe,GAAG,MAAM,KAC/B,aA+BF,CAAC;AAyCF,qBAAa,oBAAqB,SAAQ,uBAAuB;IAC7D,QAAQ,CAAC,IAAI,6BAA6B;IAE1C,cAAc,IAAI,mBAAmB;IAIrC,wBAAwB,IAAI,MAAM;IAIlC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmBnC,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM;IAKnC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM;IAM7D,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM;IAKT,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAKtC,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM;IAO5D,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;CAIzC;AAED,qBAAa,yBAA0B,SAAQ,mBAAmB,CAAC,2BAA2B,CAAC;IAC3F,MAAM,EAAE,gBAAgB,CAAC;gBAEb,WAAW,EAAE,2BAA2B;IAe9C,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA6GV,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAqC5D,SAAS,CACX,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC;IA2BtB,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EAAE,oCAAoC;IACvD,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC;IA+BtB,YAAY;;;;;IAqBZ,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAU9B"}
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.ClickhouseWarehouseClient = exports.ClickhouseSqlBuilder = exports.convertDataTypeToDimensionType = exports.ClickhouseTypes = void 0;
|
|
6
|
+
exports.ClickhouseWarehouseClient = exports.ClickhouseSqlBuilder = exports.convertDataTypeToDimensionType = exports.getClickhouseTimestampDomain = exports.ClickhouseTypes = void 0;
|
|
7
7
|
const client_1 = require("@clickhouse/client");
|
|
8
8
|
const common_1 = require("@lightdash/common");
|
|
9
9
|
const processPromisesInBatches_1 = require("../utils/processPromisesInBatches");
|
|
@@ -45,7 +45,7 @@ var ClickhouseTypes;
|
|
|
45
45
|
ClickhouseTypes["IPV4"] = "IPv4";
|
|
46
46
|
ClickhouseTypes["IPV6"] = "IPv6";
|
|
47
47
|
})(ClickhouseTypes || (exports.ClickhouseTypes = ClickhouseTypes = {}));
|
|
48
|
-
const
|
|
48
|
+
const cleanClickhouseType = (type) => {
|
|
49
49
|
// Iteratively unwrap Nullable/LowCardinality wrappers in any nesting order
|
|
50
50
|
// e.g. LowCardinality(Nullable(Int32)) -> Nullable(Int32) -> Int32
|
|
51
51
|
let cleanType = type;
|
|
@@ -57,7 +57,22 @@ const convertDataTypeToDimensionType = (type) => {
|
|
|
57
57
|
.replace(/^LowCardinality\((.+)\)$/, '$1');
|
|
58
58
|
}
|
|
59
59
|
// Strip all parenthesized arguments (handles Decimal(18,2), DateTime64(3,'UTC'), FixedString(N), etc.)
|
|
60
|
-
|
|
60
|
+
return cleanType.replace(/\(.*\)$/, '');
|
|
61
|
+
};
|
|
62
|
+
// ClickHouse has no naive timestamp type: DateTime/DateTime64 store epoch
|
|
63
|
+
// instants and their zone is display metadata, so they are always aware.
|
|
64
|
+
const getClickhouseTimestampDomain = (type) => {
|
|
65
|
+
switch (cleanClickhouseType(type)) {
|
|
66
|
+
case ClickhouseTypes.DATETIME:
|
|
67
|
+
case ClickhouseTypes.DATETIME64:
|
|
68
|
+
return 'aware';
|
|
69
|
+
default:
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
exports.getClickhouseTimestampDomain = getClickhouseTimestampDomain;
|
|
74
|
+
const convertDataTypeToDimensionType = (type) => {
|
|
75
|
+
const cleanType = cleanClickhouseType(type);
|
|
61
76
|
switch (cleanType) {
|
|
62
77
|
case ClickhouseTypes.BOOL:
|
|
63
78
|
return common_1.DimensionType.BOOLEAN;
|
|
@@ -101,6 +116,7 @@ const catalogToSchema = (results) => {
|
|
|
101
116
|
warehouseCatalog[database][tableSchema || 'default'][tableName] =
|
|
102
117
|
warehouseCatalog[database][tableSchema || 'default'][tableName] || {};
|
|
103
118
|
warehouseCatalog[database][tableSchema || 'default'][tableName][columnName] = (0, exports.convertDataTypeToDimensionType)(dataType);
|
|
119
|
+
(0, common_1.setCatalogTimestampDomain)(warehouseCatalog, database, tableSchema || 'default', tableName, columnName, (0, exports.getClickhouseTimestampDomain)(dataType));
|
|
104
120
|
});
|
|
105
121
|
});
|
|
106
122
|
return warehouseCatalog;
|
|
@@ -315,7 +331,7 @@ class ClickhouseWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
315
331
|
const { rows } = await this.runQuery(query, tags, undefined, undefined, {
|
|
316
332
|
databaseName,
|
|
317
333
|
});
|
|
318
|
-
return this.parseWarehouseCatalog(rows, exports.convertDataTypeToDimensionType);
|
|
334
|
+
return this.parseWarehouseCatalog(rows, exports.convertDataTypeToDimensionType, exports.getClickhouseTimestampDomain);
|
|
319
335
|
}
|
|
320
336
|
async getFields(tableName, schema, database, // always empty string in clickhouse
|
|
321
337
|
tags) {
|
|
@@ -336,7 +352,7 @@ class ClickhouseWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
336
352
|
databaseName: dbName,
|
|
337
353
|
tableName,
|
|
338
354
|
});
|
|
339
|
-
return this.parseWarehouseCatalog(rows, exports.convertDataTypeToDimensionType);
|
|
355
|
+
return this.parseWarehouseCatalog(rows, exports.convertDataTypeToDimensionType, exports.getClickhouseTimestampDomain);
|
|
340
356
|
}
|
|
341
357
|
async getAllTables() {
|
|
342
358
|
const databaseName = this.credentials.schema; // In clickhouse schema = database
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ConnectionOptions } from '@databricks/sql/dist/contracts/IDBSQLClient';
|
|
2
|
-
import { AnyType, CreateDatabricksCredentials, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseResults, WarehouseTypes } from '@lightdash/common';
|
|
2
|
+
import { AnyType, CreateDatabricksCredentials, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseResults, WarehouseTypes, type TimestampDomain } from '@lightdash/common';
|
|
3
3
|
import { WarehouseCatalog } from '../types';
|
|
4
4
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
5
5
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
@@ -12,6 +12,7 @@ import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
|
12
12
|
*/
|
|
13
13
|
export declare const DATABRICKS_DEFAULT_OAUTH_CLIENT_ID = "databricks-cli";
|
|
14
14
|
export declare const isDatabricksCliOAuthClientId: (clientId: string | undefined) => boolean;
|
|
15
|
+
export declare const getDatabricksTimestampDomain: (type: string) => TimestampDomain | undefined;
|
|
15
16
|
export declare class DatabricksSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
16
17
|
readonly type = WarehouseTypes.DATABRICKS;
|
|
17
18
|
getAdapterType(): SupportedDbtAdapter;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DatabricksWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DatabricksWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAqB,EACjB,iBAAiB,EACpB,MAAM,6CAA6C,CAAC;AAIrD,OAAO,EACH,OAAO,EACP,2BAA2B,EAI3B,MAAM,
|
|
1
|
+
{"version":3,"file":"DatabricksWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DatabricksWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAqB,EACjB,iBAAiB,EACpB,MAAM,6CAA6C,CAAC;AAIrD,OAAO,EACH,OAAO,EACP,2BAA2B,EAI3B,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAIhB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE;;;;;;GAMG;AACH,eAAO,MAAM,kCAAkC,mBAAmB,CAAC;AAQnE,eAAO,MAAM,4BAA4B,GACrC,UAAU,MAAM,GAAG,SAAS,KAC7B,OAAsE,CAAC;AAoJ1E,eAAO,MAAM,4BAA4B,GACrC,MAAM,MAAM,KACb,eAAe,GAAG,SASpB,CAAC;AAEF,qBAAa,oBAAqB,SAAQ,uBAAuB;IAC7D,QAAQ,CAAC,IAAI,6BAA6B;IAE1C,cAAc,IAAI,mBAAmB;IAIrC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,iBAAiB,IAAI,MAAM;IAI3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmBnC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM;IAM7D,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM;IAKT,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAKtC,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAKtC,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM;CAQ9D;AAQD,qBAAa,yBAA0B,SAAQ,mBAAmB,CAAC,2BAA2B,CAAC;IAC3F,MAAM,EAAE,MAAM,CAAC;IAEf,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,iBAAiB,EAAE,iBAAiB,CAAC;IAErC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;gBAE7B,WAAW,EAAE,2BAA2B;YAiDtC,UAAU;IAqClB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA0EV,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE;IA0ED,YAAY;;;;;IAeZ,SAAS,CACX,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC;IAsBtB,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC;CA0B/B;AAED;;GAEG;AACH,eAAO,MAAM,kCAAkC,GAC3C,MAAM,MAAM,EACZ,UAAU,MAAM,EAChB,cAAc,MAAM,KACrB,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,CA+BxD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,2BAA2B,GACpC,MAAM,MAAM,EACZ,UAAU,MAAM,EAChB,cAAc,MAAM,EACpB,eAAe,MAAM,KACtB,OAAO,CAAC;IACP,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACrB,CAqCA,CAAC"}
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.refreshDatabricksOAuthToken = exports.exchangeDatabricksOAuthCredentials = exports.DatabricksWarehouseClient = exports.DatabricksSqlBuilder = exports.isDatabricksCliOAuthClientId = exports.DATABRICKS_DEFAULT_OAUTH_CLIENT_ID = void 0;
|
|
6
|
+
exports.refreshDatabricksOAuthToken = exports.exchangeDatabricksOAuthCredentials = exports.DatabricksWarehouseClient = exports.DatabricksSqlBuilder = exports.getDatabricksTimestampDomain = exports.isDatabricksCliOAuthClientId = exports.DATABRICKS_DEFAULT_OAUTH_CLIENT_ID = void 0;
|
|
7
7
|
const sql_1 = require("@databricks/sql");
|
|
8
8
|
const TCLIService_types_1 = require("@databricks/sql/thrift/TCLIService_types");
|
|
9
9
|
const common_1 = require("@lightdash/common");
|
|
@@ -87,6 +87,7 @@ var DatabricksTypes;
|
|
|
87
87
|
DatabricksTypes["MAP"] = "MAP";
|
|
88
88
|
DatabricksTypes["CHAR"] = "CHAR";
|
|
89
89
|
DatabricksTypes["VARCHAR"] = "VARCHAR";
|
|
90
|
+
DatabricksTypes["TIMESTAMP_NTZ"] = "TIMESTAMP_NTZ";
|
|
90
91
|
})(DatabricksTypes || (DatabricksTypes = {}));
|
|
91
92
|
const normaliseDatabricksType = (type) => {
|
|
92
93
|
const r = /^[A-Z]+/;
|
|
@@ -133,6 +134,20 @@ const mapFieldType = (type) => {
|
|
|
133
134
|
return common_1.DimensionType.STRING;
|
|
134
135
|
}
|
|
135
136
|
};
|
|
137
|
+
// Classifies on the full raw string BEFORE normaliseDatabricksType, whose
|
|
138
|
+
// /^[A-Z]+/ regex truncates TIMESTAMP_NTZ to TIMESTAMP at the underscore.
|
|
139
|
+
// Bare TIMESTAMP is an instant on Databricks; NTZ is the opt-in naive type.
|
|
140
|
+
const getDatabricksTimestampDomain = (type) => {
|
|
141
|
+
switch (type.toUpperCase().replace(/\s*\(.*\)$/, '')) {
|
|
142
|
+
case DatabricksTypes.TIMESTAMP_NTZ:
|
|
143
|
+
return 'naive';
|
|
144
|
+
case DatabricksTypes.TIMESTAMP:
|
|
145
|
+
return 'aware';
|
|
146
|
+
default:
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
exports.getDatabricksTimestampDomain = getDatabricksTimestampDomain;
|
|
136
151
|
class DatabricksSqlBuilder extends WarehouseBaseSqlBuilder_1.default {
|
|
137
152
|
constructor() {
|
|
138
153
|
super(...arguments);
|
|
@@ -377,13 +392,13 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
377
392
|
}
|
|
378
393
|
const catalog = this.catalog || 'DEFAULT';
|
|
379
394
|
return results.reduce((acc, result, index) => {
|
|
380
|
-
const columns = Object.fromEntries(result.map((col) => [
|
|
381
|
-
col.COLUMN_NAME,
|
|
382
|
-
mapFieldType(col.TYPE_NAME),
|
|
383
|
-
]));
|
|
384
395
|
const { schema, table } = requests[index];
|
|
385
396
|
acc[catalog][schema] = acc[catalog][schema] || {};
|
|
386
|
-
acc[catalog][schema][table] =
|
|
397
|
+
acc[catalog][schema][table] = {};
|
|
398
|
+
result.forEach((col) => {
|
|
399
|
+
acc[catalog][schema][table][col.COLUMN_NAME] = mapFieldType(col.TYPE_NAME);
|
|
400
|
+
(0, common_1.setCatalogTimestampDomain)(acc, catalog, schema, table, col.COLUMN_NAME, (0, exports.getDatabricksTimestampDomain)(col.TYPE_NAME));
|
|
401
|
+
});
|
|
387
402
|
return acc;
|
|
388
403
|
}, { [catalog]: {} });
|
|
389
404
|
}
|
|
@@ -411,7 +426,7 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
411
426
|
ORDER BY 1,2,3
|
|
412
427
|
`;
|
|
413
428
|
const { rows } = await this.runQuery(query, tags, undefined, schema ? [schema] : undefined);
|
|
414
|
-
return this.parseWarehouseCatalog(rows, mapFieldType);
|
|
429
|
+
return this.parseWarehouseCatalog(rows, mapFieldType, exports.getDatabricksTimestampDomain);
|
|
415
430
|
}
|
|
416
431
|
async getFields(tableName, schema, database, tags) {
|
|
417
432
|
const query = `
|
|
@@ -433,7 +448,7 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
433
448
|
values.push(database);
|
|
434
449
|
}
|
|
435
450
|
const { rows } = await this.runQuery(query, tags, undefined, values);
|
|
436
|
-
return this.parseWarehouseCatalog(rows, mapFieldType);
|
|
451
|
+
return this.parseWarehouseCatalog(rows, mapFieldType, exports.getDatabricksTimestampDomain);
|
|
437
452
|
}
|
|
438
453
|
}
|
|
439
454
|
exports.DatabricksWarehouseClient = DatabricksWarehouseClient;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnyType, CreateDuckdbCredentials, CreateDuckdbMotherduckCredentials, DimensionType, Metric, SupportedDbtAdapter, WarehouseCatalog, WarehouseResults, type WarehouseQueryPhase } from '@lightdash/common';
|
|
1
|
+
import { AnyType, CreateDuckdbCredentials, CreateDuckdbMotherduckCredentials, DimensionType, Metric, SupportedDbtAdapter, WarehouseCatalog, WarehouseResults, type TimestampDomain, type WarehouseQueryPhase } from '@lightdash/common';
|
|
2
2
|
import WarehouseBaseClient from './WarehouseBaseClient';
|
|
3
3
|
import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
|
|
4
4
|
export type DuckdbS3SessionConfig = {
|
|
@@ -48,6 +48,7 @@ export type DuckdbWarehouseClientOptions = {
|
|
|
48
48
|
onQueryProfile?: (profile: DuckdbQueryProfileMetrics) => void;
|
|
49
49
|
};
|
|
50
50
|
export declare const mapFieldTypeFromTypeId: (typeId: number) => DimensionType;
|
|
51
|
+
export declare const getDuckdbTimestampDomainFromString: (typeName: string) => TimestampDomain | undefined;
|
|
51
52
|
export declare class DuckdbSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
52
53
|
getAdapterType(): SupportedDbtAdapter;
|
|
53
54
|
getFloatingType(): string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,uBAAuB,EAEvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,uBAAuB,EAEvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAK3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA4DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAG9D,MAAM,MAAM,wBAAwB,GAAG,mBAAmB,CAAC;AAE3D,MAAM,MAAM,4BAA4B,GAAG;IACvC,2FAA2F;IAC3F,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACjE,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,GAC3C,UAAU,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAgED,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;gBAGN,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B;IAwF1C,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,CAAC,EAAE,2BAA2B,EACzC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB;IAIxB,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,kBAAkB;mBAQL,cAAc;IAWnC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,kCAAkC;mBAUlC,qBAAqB;mBAqFrB,uBAAuB;mBAgBvB,yBAAyB;IA6E9C,OAAO,CAAC,MAAM,CAAC,mBAAmB;IAoBlC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI;IAKzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAqC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IAqDtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IAkC/B,OAAO,CAAC,iBAAiB;IAMzB;;;;;;OAMG;YACW,WAAW;IAkBzB,+DAA+D;YACjD,iBAAiB;IAyC/B,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAM/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAUnC,OAAO,CAAC,MAAM,CAAC,yBAAyB;YAkB1B,eAAe;YA+Bf,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC;IA2DV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAoBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC;IA8CtB,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAuBK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAU/B"}
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.DuckdbWarehouseClient = exports.DuckdbSqlBuilder = exports.mapFieldTypeFromTypeId = void 0;
|
|
6
|
+
exports.DuckdbWarehouseClient = exports.DuckdbSqlBuilder = exports.getDuckdbTimestampDomainFromString = exports.mapFieldTypeFromTypeId = void 0;
|
|
7
7
|
const node_api_1 = require("@duckdb/node-api");
|
|
8
8
|
const common_1 = require("@lightdash/common");
|
|
9
9
|
const crypto_1 = require("crypto");
|
|
@@ -45,6 +45,16 @@ const mapFieldTypeFromTypeId = (typeId) => {
|
|
|
45
45
|
}
|
|
46
46
|
};
|
|
47
47
|
exports.mapFieldTypeFromTypeId = mapFieldTypeFromTypeId;
|
|
48
|
+
const getDuckdbTimestampDomainFromString = (typeName) => {
|
|
49
|
+
const upper = typeName.toUpperCase().replace(/\(\d+\)/, '');
|
|
50
|
+
if (upper === 'TIMESTAMP WITH TIME ZONE' || upper === 'TIMESTAMPTZ')
|
|
51
|
+
return 'aware';
|
|
52
|
+
// TIMESTAMP and its precision variants (TIMESTAMP_S/_MS/_NS)
|
|
53
|
+
if (upper.startsWith('TIMESTAMP'))
|
|
54
|
+
return 'naive';
|
|
55
|
+
return undefined;
|
|
56
|
+
};
|
|
57
|
+
exports.getDuckdbTimestampDomainFromString = getDuckdbTimestampDomainFromString;
|
|
48
58
|
const mapFieldTypeFromString = (typeName) => {
|
|
49
59
|
const upper = typeName.toUpperCase();
|
|
50
60
|
if (upper === 'DATE')
|
|
@@ -1106,12 +1116,14 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1106
1116
|
if (!catalog[ref.database][ref.schema]) {
|
|
1107
1117
|
catalog[ref.database][ref.schema] = {};
|
|
1108
1118
|
}
|
|
1109
|
-
catalog[ref.database][ref.schema][ref.table] =
|
|
1119
|
+
catalog[ref.database][ref.schema][ref.table] = {};
|
|
1120
|
+
rows.forEach((row) => {
|
|
1110
1121
|
const colName = row.column_name;
|
|
1111
1122
|
const colType = row.data_type;
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1123
|
+
catalog[ref.database][ref.schema][ref.table][colName] =
|
|
1124
|
+
mapFieldTypeFromString(colType);
|
|
1125
|
+
(0, common_1.setCatalogTimestampDomain)(catalog, ref.database, ref.schema, ref.table, colName, (0, exports.getDuckdbTimestampDomainFromString)(colType));
|
|
1126
|
+
});
|
|
1115
1127
|
}
|
|
1116
1128
|
}
|
|
1117
1129
|
/* eslint-enable no-await-in-loop */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnyType, CreatePostgresCredentials, CreatePostgresLikeCredentials,
|
|
1
|
+
import { AnyType, CreatePostgresCredentials, CreatePostgresLikeCredentials, Metric, SupportedDbtAdapter, WarehouseCatalog, WarehouseQueryError, WarehouseResults, WarehouseTypes, type TimestampDomain, type WarehouseQueryPhase } from '@lightdash/common';
|
|
2
2
|
import * as pg from 'pg';
|
|
3
3
|
import { QueryResult } from 'pg';
|
|
4
4
|
import './pgProtocolGuard';
|
|
@@ -44,8 +44,10 @@ export declare enum PostgresTypes {
|
|
|
44
44
|
TIME_WITHOUT_TIME_ZONE = "time without time zone",
|
|
45
45
|
TIMESTAMP = "timestamp",
|
|
46
46
|
TIMESTAMP_TZ = "timestamptz",
|
|
47
|
-
TIMESTAMP_WITHOUT_TIME_ZONE = "timestamp without time zone"
|
|
47
|
+
TIMESTAMP_WITHOUT_TIME_ZONE = "timestamp without time zone",
|
|
48
|
+
TIMESTAMP_WITH_TIME_ZONE = "timestamp with time zone"
|
|
48
49
|
}
|
|
50
|
+
export declare const getPostgresTimestampDomain: (type: string) => TimestampDomain | undefined;
|
|
49
51
|
export declare class PostgresSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
50
52
|
type: WarehouseTypes;
|
|
51
53
|
getAdapterType(): SupportedDbtAdapter;
|
|
@@ -58,9 +60,7 @@ export declare class PostgresClient<T extends CreatePostgresLikeCredentials> ext
|
|
|
58
60
|
config: pg.PoolConfig;
|
|
59
61
|
constructor(credentials: T, config: pg.PoolConfig);
|
|
60
62
|
private getSQLWithMetadata;
|
|
61
|
-
static convertQueryResultFields(fields: QueryResult<AnyType>['fields']):
|
|
62
|
-
type: DimensionType;
|
|
63
|
-
}>;
|
|
63
|
+
static convertQueryResultFields(fields: QueryResult<AnyType>['fields']): WarehouseResults['fields'];
|
|
64
64
|
private static getNoticeError;
|
|
65
65
|
streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options: {
|
|
66
66
|
values?: AnyType[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,
|
|
1
|
+
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAG7B,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAc,WAAW,EAAS,MAAM,IAAI,CAAC;AAKpD,OAAO,mBAAmB,CAAC;AAE3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,aAAa;IACrB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,WAAW,gBAAgB;IAC3B,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,gBAAgB,qBAAqB;IACrC,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,WAAW;IAClB,sBAAsB,2BAA2B;IACjD,SAAS,cAAc;IACvB,YAAY,gBAAgB;IAC5B,2BAA2B,gCAAgC;IAC3D,wBAAwB,6BAA6B;CACxD;AA4CD,eAAO,MAAM,0BAA0B,GACnC,MAAM,MAAM,KACb,eAAe,GAAG,SAWpB,CAAC;AA6CF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,IAAI,iBAA2B;IAE/B,cAAc,IAAI,mBAAmB;IAIrC,wBAAwB,IAAI,MAAM;IAIlC,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmBnC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAejD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAED,qBAAa,cAAc,CACvB,CAAC,SAAS,6BAA6B,CACzC,SAAQ,mBAAmB,CAAC,CAAC,CAAC;IAC5B,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;gBAEV,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU;IAKjD,OAAO,CAAC,kBAAkB;IAQ1B,MAAM,CAAC,wBAAwB,CAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GACvC,gBAAgB,CAAC,QAAQ,CAAC;IAS7B,OAAO,CAAC,MAAM,CAAC,cAAc;IAavB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC;IA4OV,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE;IA4GD,YAAY;;;;;IAyBZ,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC;IA4B5B,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,EAAE,KAAK,GAAE,MAAW;CAuDzD;AA0CD,qBAAa,uBAAwB,SAAQ,cAAc,CAAC,yBAAyB,CAAC;gBACtE,WAAW,EAAE,yBAAyB;CAWrD"}
|
|
@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.PostgresWarehouseClient = exports.PostgresClient = exports.PostgresSqlBuilder = exports.PostgresTypes = void 0;
|
|
39
|
+
exports.PostgresWarehouseClient = exports.PostgresClient = exports.PostgresSqlBuilder = exports.getPostgresTimestampDomain = exports.PostgresTypes = void 0;
|
|
40
40
|
const common_1 = require("@lightdash/common");
|
|
41
41
|
const fs_1 = require("fs");
|
|
42
42
|
const path_1 = __importDefault(require("path"));
|
|
@@ -94,6 +94,7 @@ var PostgresTypes;
|
|
|
94
94
|
PostgresTypes["TIMESTAMP"] = "timestamp";
|
|
95
95
|
PostgresTypes["TIMESTAMP_TZ"] = "timestamptz";
|
|
96
96
|
PostgresTypes["TIMESTAMP_WITHOUT_TIME_ZONE"] = "timestamp without time zone";
|
|
97
|
+
PostgresTypes["TIMESTAMP_WITH_TIME_ZONE"] = "timestamp with time zone";
|
|
97
98
|
})(PostgresTypes || (exports.PostgresTypes = PostgresTypes = {}));
|
|
98
99
|
const mapFieldType = (type) => {
|
|
99
100
|
switch (type) {
|
|
@@ -126,6 +127,7 @@ const mapFieldType = (type) => {
|
|
|
126
127
|
case PostgresTypes.TIMESTAMP_TZ:
|
|
127
128
|
case PostgresTypes.TIME_WITHOUT_TIME_ZONE:
|
|
128
129
|
case PostgresTypes.TIMESTAMP_WITHOUT_TIME_ZONE:
|
|
130
|
+
case PostgresTypes.TIMESTAMP_WITH_TIME_ZONE:
|
|
129
131
|
return common_1.DimensionType.TIMESTAMP;
|
|
130
132
|
case PostgresTypes.BOOLEAN:
|
|
131
133
|
case PostgresTypes.BOOL:
|
|
@@ -134,6 +136,20 @@ const mapFieldType = (type) => {
|
|
|
134
136
|
return common_1.DimensionType.STRING;
|
|
135
137
|
}
|
|
136
138
|
};
|
|
139
|
+
// Strips the precision that format_type emits, e.g. `timestamp(3) with time zone`
|
|
140
|
+
const getPostgresTimestampDomain = (type) => {
|
|
141
|
+
switch (type.replace(/\(\d+\)/, '').replace(/\s{2,}/g, ' ')) {
|
|
142
|
+
case PostgresTypes.TIMESTAMP:
|
|
143
|
+
case PostgresTypes.TIMESTAMP_WITHOUT_TIME_ZONE:
|
|
144
|
+
return 'naive';
|
|
145
|
+
case PostgresTypes.TIMESTAMP_TZ:
|
|
146
|
+
case PostgresTypes.TIMESTAMP_WITH_TIME_ZONE:
|
|
147
|
+
return 'aware';
|
|
148
|
+
default:
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
};
|
|
152
|
+
exports.getPostgresTimestampDomain = getPostgresTimestampDomain;
|
|
137
153
|
const { builtins } = pg.types;
|
|
138
154
|
const POSTGRES_NAME_TOO_LONG_SQLSTATE = '42622';
|
|
139
155
|
// Server-side ceiling for a single streamed query, bounded just under the
|
|
@@ -483,6 +499,7 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
483
499
|
acc[table_catalog][table_schema][table_name] || {};
|
|
484
500
|
acc[table_catalog][table_schema][table_name][column_name] =
|
|
485
501
|
mapFieldType(data_type);
|
|
502
|
+
(0, common_1.setCatalogTimestampDomain)(acc, table_catalog, table_schema, table_name, column_name, (0, exports.getPostgresTimestampDomain)(data_type));
|
|
486
503
|
}
|
|
487
504
|
return acc;
|
|
488
505
|
}, {});
|
|
@@ -527,7 +544,7 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
527
544
|
values.push(database);
|
|
528
545
|
}
|
|
529
546
|
const { rows } = await this.runQuery(query, tags, undefined, values);
|
|
530
|
-
return this.parseWarehouseCatalog(rows, mapFieldType);
|
|
547
|
+
return this.parseWarehouseCatalog(rows, mapFieldType, exports.getPostgresTimestampDomain);
|
|
531
548
|
}
|
|
532
549
|
parseError(error, query = '') {
|
|
533
550
|
// getErrorLineAndCharPosition is a helper function to get the line and character position of the error
|
|
@@ -141,7 +141,7 @@ describe('PostgresWarehouseClient', () => {
|
|
|
141
141
|
on: vi.fn(async () => undefined),
|
|
142
142
|
};
|
|
143
143
|
});
|
|
144
|
-
expect(await warehouse.getCatalog(WarehouseClient_mock_1.config)).toEqual(WarehouseClient_mock_1.
|
|
144
|
+
expect(await warehouse.getCatalog(WarehouseClient_mock_1.config)).toEqual(WarehouseClient_mock_1.expectedWarehouseSchemaWithNaiveTimestamp);
|
|
145
145
|
});
|
|
146
146
|
it('expect empty catalog when dbt project has no references', async () => {
|
|
147
147
|
const warehouse = new PostgresWarehouseClient_1.PostgresWarehouseClient(PostgresWarehouseClient_mock_1.credentials);
|