@lightdash/warehouses 1.107.1 → 1.108.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.js +14 -5
- package/dist/warehouseClients/BigqueryWarehouseClient.mock.d.ts.map +1 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.mock.js +23 -7
- package/dist/warehouseClients/BigqueryWarehouseClient.test.js +14 -3
- package/package.json +2 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAEP,eAAe,EACf,eAAe,EACf,6BAA6B,EAC7B,yBAAyB,EACzB,aAAa,EAEb,MAAM,EAEN,eAAe,EAKf,mBAAmB,EACnB,gBAAgB,EAEhB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAEP,eAAe,EACf,eAAe,EACf,6BAA6B,EAC7B,yBAAyB,EACzB,aAAa,EAEb,MAAM,EAEN,eAAe,EAKf,mBAAmB,EACnB,gBAAgB,EAEhB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACH,gBAAgB,EAChB,0BAA0B,EAC1B,8BAA8B,EAEjC,MAAM,UAAU,CAAC;AAOlB,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,iBAAiB;IACzB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,OAAO,YAAY;IACnB,UAAU,eAAe;IACzB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,KAAK,UAAU;CAClB;AAiDD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,GAAG,SAAS,KACzB,eAAe,GAAG,SASpB,CAAC;AA0BF,KAAK,WAAW,GAAG;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;CAC1B,CAAC;AAEF,KAAK,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;AAehF,KAAK,aAAa,GAAG;IACjB,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;CAClC,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,QAAQ,CAAC,IAAI,2BAA2B;IAExC,cAAc,IAAI,mBAAmB,CAEpC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAWhD;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGlC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAM1D;CACJ;AAED,qBAAa,uBAAwB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACvF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAM;IAExC,MAAM,EAAE,QAAQ,CAAC;IAEjB,YAAY,WAAW,EAAE,yBAAyB,EAyBjD;IAED,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAE7D;IAED;;;;;;;;;;OAUG;IACH,MAAM,CAAC,wBAAwB,CAC3B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAoBpC;IAED,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,SAAS;;OAYtE;YAEa,SAAS;YA+BT,MAAM;YAWN,qBAAqB;YAUrB,aAAa;IAoCrB,WAAW,CACb,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC,CAsBf;IAED,OAAa,gBAAgB,CACzB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAShD;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,6BAgDN;IAEK,YAAY;;;;;SA6DjB;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAmB3B;IAED,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,GAAE,MAAW,uBA+FzD;IAEK,iBAAiB,CACnB,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,8BAA8B,EACvD,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GACV,OAAO,CAAC,0BAA0B,CAAC,CA8DrC;YAEa,kBAAkB;IAWhC,OAAa,YAAY,CACrB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,cAAc,GAAE;QACZ,WAAW,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KASvD,GACH,OAAO,CAAC,eAAe,EAAE,CAAC,CA0B5B;IAED,OAAa,wBAAwB,CACjC,QAAQ,EAAE,eAAe,EAAE,EAC3B,YAAY,EAAE,MAAM,EACpB,qBAAqB,GAAE,CAAC,SAAS,EAAE,MAAM,KAAK;QAC1C,WAAW,EAAE,CACT,OAAO,EAAE,kBAAkB,KAC1B,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/B,KAAK,EAAE,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAgBpC,GACF,OAAO,CAAC,6BAA6B,CAAC,CAuFxC;IAED,OAAa,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA0DxE;CACJ"}
|
|
@@ -6,7 +6,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.BigqueryWarehouseClient = exports.BigquerySqlBuilder = exports.getBigqueryTimestampDomain = exports.BigqueryFieldType = void 0;
|
|
7
7
|
const bigquery_1 = require("@google-cloud/bigquery");
|
|
8
8
|
const common_1 = require("@lightdash/common");
|
|
9
|
-
const big_js_1 = __importDefault(require("big.js"));
|
|
10
9
|
const stream_1 = require("stream");
|
|
11
10
|
const coerceTagToString_1 = require("../utils/coerceTagToString");
|
|
12
11
|
const processPromisesInBatches_1 = require("../utils/processPromisesInBatches");
|
|
@@ -36,6 +35,18 @@ var BigqueryFieldType;
|
|
|
36
35
|
BigqueryFieldType["STRUCT"] = "STRUCT";
|
|
37
36
|
BigqueryFieldType["ARRAY"] = "ARRAY";
|
|
38
37
|
})(BigqueryFieldType || (exports.BigqueryFieldType = BigqueryFieldType = {}));
|
|
38
|
+
const isBigqueryDecimal = (value) => typeof value === 'object' &&
|
|
39
|
+
value !== null &&
|
|
40
|
+
'e' in value &&
|
|
41
|
+
Number.isInteger(value.e) &&
|
|
42
|
+
's' in value &&
|
|
43
|
+
(value.s === 1 || value.s === -1) &&
|
|
44
|
+
'toFixed' in value &&
|
|
45
|
+
typeof value.toFixed === 'function' &&
|
|
46
|
+
'c' in value &&
|
|
47
|
+
Array.isArray(value.c) &&
|
|
48
|
+
value.c.length > 0 &&
|
|
49
|
+
value.c.every((digit) => typeof digit === 'number');
|
|
39
50
|
const parseCell = (cell) => {
|
|
40
51
|
if (cell === undefined ||
|
|
41
52
|
cell === null ||
|
|
@@ -49,10 +60,8 @@ const parseCell = (cell) => {
|
|
|
49
60
|
cell instanceof bigquery_1.BigQueryTime) {
|
|
50
61
|
return new Date(cell.value);
|
|
51
62
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
if (cell instanceof big_js_1.default) {
|
|
55
|
-
return Number(cell);
|
|
63
|
+
if (isBigqueryDecimal(cell)) {
|
|
64
|
+
return Number(cell.toFixed());
|
|
56
65
|
}
|
|
57
66
|
return `${cell}`;
|
|
58
67
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BigqueryWarehouseClient.mock.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.mock.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"BigqueryWarehouseClient.mock.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.mock.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,yBAAyB,EAE5B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,eAAO,MAAM,WAAW,EAAE,yBAUzB,CAAC;AAyCF,eAAO,MAAM,gBAAgB;IACzB,EAAE;IACF,QAAQ;QAAI,SAAS;;IACrB,WAAW;;;;;;;;CACd,CAAC;AAwBF,eAAO,MAAM,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAUzC,CAAC;AAEF,QAAA,MAAM,aAAa,gBAOb,CAAC;AAEP,eAAO,MAAM,iBAAiB;;;;;;;;;;GAK7B,CAAC"}
|
|
@@ -1,12 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
3
|
exports.createJobResponse = exports.rows = exports.getTableResponse = exports.credentials = void 0;
|
|
7
4
|
const bigquery_1 = require("@google-cloud/bigquery");
|
|
8
5
|
const common_1 = require("@lightdash/common");
|
|
9
|
-
const big_js_1 = __importDefault(require("big.js"));
|
|
10
6
|
const stream_1 = require("stream");
|
|
11
7
|
const BigqueryWarehouseClient_1 = require("./BigqueryWarehouseClient");
|
|
12
8
|
exports.credentials = {
|
|
@@ -31,6 +27,10 @@ const metadata = {
|
|
|
31
27
|
name: 'myNumberColumn',
|
|
32
28
|
type: BigqueryWarehouseClient_1.BigqueryFieldType.NUMERIC,
|
|
33
29
|
},
|
|
30
|
+
{
|
|
31
|
+
name: 'myBigNumberColumn',
|
|
32
|
+
type: BigqueryWarehouseClient_1.BigqueryFieldType.BIGNUMERIC,
|
|
33
|
+
},
|
|
34
34
|
{
|
|
35
35
|
name: 'myDateColumn',
|
|
36
36
|
type: BigqueryWarehouseClient_1.BigqueryFieldType.DATE,
|
|
@@ -59,16 +59,32 @@ exports.getTableResponse = {
|
|
|
59
59
|
bigQuery: { projectId: 'myDatabase' },
|
|
60
60
|
getMetadata: vi.fn(() => [metadata]),
|
|
61
61
|
};
|
|
62
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
63
|
+
const [bigqueryDecimalValues] = bigquery_1.BigQuery.mergeSchemaWithRows_({
|
|
64
|
+
fields: [
|
|
65
|
+
{
|
|
66
|
+
name: 'myNumberColumn',
|
|
67
|
+
type: BigqueryWarehouseClient_1.BigqueryFieldType.NUMERIC,
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'myBigNumberColumn',
|
|
71
|
+
type: BigqueryWarehouseClient_1.BigqueryFieldType.BIGNUMERIC,
|
|
72
|
+
},
|
|
73
|
+
],
|
|
74
|
+
}, [
|
|
75
|
+
{
|
|
76
|
+
f: [{ v: '100.25' }, { v: '200.5' }],
|
|
77
|
+
},
|
|
78
|
+
], { wrapIntegers: false });
|
|
62
79
|
exports.rows = [
|
|
63
80
|
{
|
|
64
81
|
myStringColumn: 'string value',
|
|
65
|
-
|
|
66
|
-
myNumberColumn: new big_js_1.default('100'),
|
|
82
|
+
...bigqueryDecimalValues,
|
|
67
83
|
myDateColumn: new bigquery_1.BigQueryDate('2021-03-10'),
|
|
68
84
|
myTimestampColumn: new bigquery_1.BigQueryTimestamp('1990-03-02T08:30:00.010Z'),
|
|
69
85
|
myBooleanColumn: false,
|
|
70
86
|
myArrayColumn: ['1', '2', '3'],
|
|
71
|
-
myObjectColumn: { test: '1' },
|
|
87
|
+
myObjectColumn: { test: '1', toFixed: () => '123' },
|
|
72
88
|
},
|
|
73
89
|
];
|
|
74
90
|
const mockStreamRow = () => new stream_1.Readable({
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const bigquery_1 = require("@google-cloud/bigquery");
|
|
4
|
+
const common_1 = require("@lightdash/common");
|
|
4
5
|
const BigqueryWarehouseClient_1 = require("./BigqueryWarehouseClient");
|
|
5
6
|
const BigqueryWarehouseClient_mock_1 = require("./BigqueryWarehouseClient.mock");
|
|
6
7
|
const WarehouseClient_mock_1 = require("./WarehouseClient.mock");
|
|
@@ -9,8 +10,15 @@ describe('BigqueryWarehouseClient', () => {
|
|
|
9
10
|
const warehouse = new BigqueryWarehouseClient_1.BigqueryWarehouseClient(BigqueryWarehouseClient_mock_1.credentials);
|
|
10
11
|
warehouse.client.createQueryJob = vi.fn(() => BigqueryWarehouseClient_mock_1.createJobResponse);
|
|
11
12
|
const results = await warehouse.runQuery('fake sql');
|
|
12
|
-
expect(results.fields).toEqual(
|
|
13
|
-
|
|
13
|
+
expect(results.fields).toEqual({
|
|
14
|
+
...WarehouseClient_mock_1.expectedFields,
|
|
15
|
+
myBigNumberColumn: { type: 'number' },
|
|
16
|
+
});
|
|
17
|
+
expect(results.rows[0]).toEqual({
|
|
18
|
+
...WarehouseClient_mock_1.expectedRow,
|
|
19
|
+
myNumberColumn: 100.25,
|
|
20
|
+
myBigNumberColumn: 200.5,
|
|
21
|
+
});
|
|
14
22
|
expect(warehouse.client.createQueryJob).toHaveBeenCalledTimes(1);
|
|
15
23
|
});
|
|
16
24
|
it('expect createQueryJob to set the time_zone connection property when a timezone is passed', async () => {
|
|
@@ -37,7 +45,10 @@ describe('BigqueryWarehouseClient', () => {
|
|
|
37
45
|
.mockImplementationOnce(() => BigqueryWarehouseClient_mock_1.getTableResponse);
|
|
38
46
|
bigquery_1.Dataset.prototype.table = getTableMock;
|
|
39
47
|
const warehouse = new BigqueryWarehouseClient_1.BigqueryWarehouseClient(BigqueryWarehouseClient_mock_1.credentials);
|
|
40
|
-
|
|
48
|
+
const expectedCatalog = structuredClone(WarehouseClient_mock_1.expectedWarehouseSchemaWithAwareTimestamp);
|
|
49
|
+
expectedCatalog.myDatabase.mySchema.myTable.myBigNumberColumn =
|
|
50
|
+
common_1.DimensionType.NUMBER;
|
|
51
|
+
expect(await warehouse.getCatalog(WarehouseClient_mock_1.config)).toEqual(expectedCatalog);
|
|
41
52
|
expect(getTableMock).toHaveBeenCalledTimes(1);
|
|
42
53
|
expect(BigqueryWarehouseClient_mock_1.getTableResponse.getMetadata).toHaveBeenCalledTimes(1);
|
|
43
54
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.108.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,6 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"big.js": "6.1.1",
|
|
25
24
|
"node-fetch": "2.7.0",
|
|
26
25
|
"pg": "8.13.1",
|
|
27
26
|
"pg-cursor": "2.10.0",
|
|
@@ -29,10 +28,9 @@
|
|
|
29
28
|
"snowflake-sdk": "2.3.4",
|
|
30
29
|
"ssh2": "1.14.0",
|
|
31
30
|
"trino-client": "0.2.9",
|
|
32
|
-
"@lightdash/common": "1.
|
|
31
|
+
"@lightdash/common": "1.108.1"
|
|
33
32
|
},
|
|
34
33
|
"devDependencies": {
|
|
35
|
-
"@types/big.js": "6.2.2",
|
|
36
34
|
"oxlint": "1.75.0",
|
|
37
35
|
"oxlint-tsgolint": "7.0.2001",
|
|
38
36
|
"@types/node-fetch": "2.6.13",
|