@lightdash/warehouses 2.228.1 → 2.229.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo.build +1 -0
- package/package.json +4 -4
- package/dist/.tsbuildinfo +0 -1
- package/dist/ssh/sshTunnelFailure.test.d.ts +0 -2
- package/dist/ssh/sshTunnelFailure.test.d.ts.map +0 -1
- package/dist/ssh/sshTunnelFailure.test.js +0 -90
- package/dist/validateWarehouseColumnReferences.test.d.ts +0 -2
- package/dist/validateWarehouseColumnReferences.test.d.ts.map +0 -1
- package/dist/validateWarehouseColumnReferences.test.js +0 -168
- package/dist/warehouseClients/AthenaWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/AthenaWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/AthenaWarehouseClient.test.js +0 -261
- package/dist/warehouseClients/BigqueryWarehouseClient.mock.d.ts +0 -83
- package/dist/warehouseClients/BigqueryWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.mock.js +0 -130
- package/dist/warehouseClients/BigqueryWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/BigqueryWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/BigqueryWarehouseClient.test.js +0 -493
- package/dist/warehouseClients/ClickhouseWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/ClickhouseWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/ClickhouseWarehouseClient.test.js +0 -115
- package/dist/warehouseClients/DatabricksWarehouseClient.mock.d.ts +0 -17
- package/dist/warehouseClients/DatabricksWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.mock.js +0 -111
- package/dist/warehouseClients/DatabricksWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/DatabricksWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.test.js +0 -592
- package/dist/warehouseClients/DatabricksWarehouseStartupRetry.test.d.ts +0 -2
- package/dist/warehouseClients/DatabricksWarehouseStartupRetry.test.d.ts.map +0 -1
- package/dist/warehouseClients/DatabricksWarehouseStartupRetry.test.js +0 -129
- package/dist/warehouseClients/DuckdbWarehouseClient.embedded.test.d.ts +0 -2
- package/dist/warehouseClients/DuckdbWarehouseClient.embedded.test.d.ts.map +0 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.embedded.test.js +0 -92
- package/dist/warehouseClients/DuckdbWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/DuckdbWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +0 -1749
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckErrorClassifier.test.js +0 -31
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.js +0 -52
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.js +0 -172
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.js +0 -192
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.test.js +0 -302
- package/dist/warehouseClients/PostgresWarehouseClient.mock.d.ts +0 -8
- package/dist/warehouseClients/PostgresWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/PostgresWarehouseClient.mock.js +0 -66
- package/dist/warehouseClients/PostgresWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/PostgresWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/PostgresWarehouseClient.test.js +0 -371
- package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/RedshiftWarehouseClient.test.js +0 -126
- package/dist/warehouseClients/SnowflakeWarehouseClient.mock.d.ts +0 -41
- package/dist/warehouseClients/SnowflakeWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.mock.js +0 -184
- package/dist/warehouseClients/SnowflakeWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/SnowflakeWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.test.js +0 -777
- package/dist/warehouseClients/TrinoWarehouseClient.mock.d.ts +0 -23
- package/dist/warehouseClients/TrinoWarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/TrinoWarehouseClient.mock.js +0 -102
- package/dist/warehouseClients/TrinoWarehouseClient.test.d.ts +0 -2
- package/dist/warehouseClients/TrinoWarehouseClient.test.d.ts.map +0 -1
- package/dist/warehouseClients/TrinoWarehouseClient.test.js +0 -230
- package/dist/warehouseClients/WarehouseClient.mock.d.ts +0 -13
- package/dist/warehouseClients/WarehouseClient.mock.d.ts.map +0 -1
- package/dist/warehouseClients/WarehouseClient.mock.js +0 -51
- package/dist/warehouseClients/numericKinds.test.d.ts +0 -2
- package/dist/warehouseClients/numericKinds.test.d.ts.map +0 -1
- package/dist/warehouseClients/numericKinds.test.js +0 -168
- package/dist/warehouseClients/pgProtocolGuard.integration.test.d.ts +0 -2
- package/dist/warehouseClients/pgProtocolGuard.integration.test.d.ts.map +0 -1
- package/dist/warehouseClients/pgProtocolGuard.integration.test.js +0 -93
- package/dist/warehouseClients/pgProtocolGuard.test.d.ts +0 -2
- package/dist/warehouseClients/pgProtocolGuard.test.d.ts.map +0 -1
- package/dist/warehouseClients/pgProtocolGuard.test.js +0 -48
- package/dist/warehouseClients/redshiftIamCredentials.test.d.ts +0 -2
- package/dist/warehouseClients/redshiftIamCredentials.test.d.ts.map +0 -1
- package/dist/warehouseClients/redshiftIamCredentials.test.js +0 -210
- package/dist/warehouseClients/timestampDomain.test.d.ts +0 -2
- package/dist/warehouseClients/timestampDomain.test.d.ts.map +0 -1
- package/dist/warehouseClients/timestampDomain.test.js +0 -101
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sshTunnelFailure.test.d.ts","sourceRoot":"","sources":["../../src/ssh/sshTunnelFailure.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const vitest_1 = require("vitest");
|
|
4
|
-
const sshTunnelFailure_1 = require("./sshTunnelFailure");
|
|
5
|
-
const err = (message, extra = {}) => Object.assign(new Error(message), extra);
|
|
6
|
-
const before = { tcpConnected: false, handshakeCompleted: false };
|
|
7
|
-
const afterTcp = { tcpConnected: true, handshakeCompleted: false };
|
|
8
|
-
const afterHandshake = { tcpConnected: true, handshakeCompleted: true };
|
|
9
|
-
(0, vitest_1.describe)('classifySshClientError', () => {
|
|
10
|
-
(0, vitest_1.it)('maps DNS failures to resolve', () => {
|
|
11
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.classifySshClientError)(err('getaddrinfo ENOTFOUND bastion.example', {
|
|
12
|
-
code: 'ENOTFOUND',
|
|
13
|
-
}), before)).toBe('resolve');
|
|
14
|
-
});
|
|
15
|
-
(0, vitest_1.it)('maps anything before the TCP connect to tcp, including a dropped SYN', () => {
|
|
16
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.classifySshClientError)(err('connect ETIMEDOUT 203.0.113.10:22 after 10000ms', {
|
|
17
|
-
code: 'ETIMEDOUT',
|
|
18
|
-
}), before)).toBe('tcp');
|
|
19
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.classifySshClientError)(err('connect ECONNREFUSED 127.0.0.1:2222', {
|
|
20
|
-
code: 'ECONNREFUSED',
|
|
21
|
-
}), before)).toBe('tcp');
|
|
22
|
-
});
|
|
23
|
-
(0, vitest_1.it)('maps a ready timeout after the TCP connect to handshake', () => {
|
|
24
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.classifySshClientError)(err('Timed out while waiting for handshake', {
|
|
25
|
-
level: 'client-timeout',
|
|
26
|
-
}), afterTcp)).toBe('handshake');
|
|
27
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.classifySshClientError)(err('Connection lost before handshake', {
|
|
28
|
-
level: 'client-socket',
|
|
29
|
-
}), afterTcp)).toBe('handshake');
|
|
30
|
-
});
|
|
31
|
-
(0, vitest_1.it)('maps rejected keys to auth', () => {
|
|
32
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.classifySshClientError)(err('All configured authentication methods failed', {
|
|
33
|
-
level: 'client-authentication',
|
|
34
|
-
}), afterHandshake)).toBe('auth');
|
|
35
|
-
});
|
|
36
|
-
(0, vitest_1.it)('treats a socket reset after the handshake as auth, not tcp', () => {
|
|
37
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.classifySshClientError)(err('read ECONNRESET', {
|
|
38
|
-
code: 'ECONNRESET',
|
|
39
|
-
level: 'client-socket',
|
|
40
|
-
}), afterHandshake)).toBe('auth');
|
|
41
|
-
});
|
|
42
|
-
});
|
|
43
|
-
(0, vitest_1.describe)('describeSshTunnelFailure', () => {
|
|
44
|
-
const base = {
|
|
45
|
-
sshHost: '203.0.113.10',
|
|
46
|
-
sshPort: 22,
|
|
47
|
-
sshUser: 'lightdash',
|
|
48
|
-
databaseHost: 'redshift.internal',
|
|
49
|
-
databasePort: 5439,
|
|
50
|
-
staticIp: '35.1.2.3',
|
|
51
|
-
cause: 'connect ETIMEDOUT',
|
|
52
|
-
};
|
|
53
|
-
(0, vitest_1.it)('tells the bastion admin which IP to allow on tcp failures', () => {
|
|
54
|
-
const message = (0, sshTunnelFailure_1.describeSshTunnelFailure)({ ...base, stage: 'tcp' });
|
|
55
|
-
(0, vitest_1.expect)(message).toContain('Could not reach 203.0.113.10 on port 22');
|
|
56
|
-
(0, vitest_1.expect)(message).toContain('from Lightdash (35.1.2.3)');
|
|
57
|
-
(0, vitest_1.expect)(message).toContain('Allow inbound SSH from 35.1.2.3');
|
|
58
|
-
});
|
|
59
|
-
(0, vitest_1.it)('omits the IP when the instance has none', () => {
|
|
60
|
-
const message = (0, sshTunnelFailure_1.describeSshTunnelFailure)({
|
|
61
|
-
...base,
|
|
62
|
-
staticIp: null,
|
|
63
|
-
stage: 'tcp',
|
|
64
|
-
});
|
|
65
|
-
(0, vitest_1.expect)(message).not.toContain('35.1.2.3');
|
|
66
|
-
(0, vitest_1.expect)(message).toContain("Lightdash's IP");
|
|
67
|
-
});
|
|
68
|
-
(0, vitest_1.it)('names the user on auth failures', () => {
|
|
69
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.describeSshTunnelFailure)({ ...base, stage: 'auth' })).toContain('rejected the key for user "lightdash"');
|
|
70
|
-
});
|
|
71
|
-
(0, vitest_1.it)('names the database endpoint on forward failures', () => {
|
|
72
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.describeSshTunnelFailure)({ ...base, stage: 'forward' })).toContain('redshift.internal:5439');
|
|
73
|
-
});
|
|
74
|
-
});
|
|
75
|
-
(0, vitest_1.describe)('sshClientErrorMessage', () => {
|
|
76
|
-
(0, vitest_1.it)('unpacks the empty-message AggregateError Node emits for a refused port', () => {
|
|
77
|
-
const aggregate = err('', {
|
|
78
|
-
code: 'ECONNREFUSED',
|
|
79
|
-
errors: [
|
|
80
|
-
new Error('connect ECONNREFUSED ::1:2299'),
|
|
81
|
-
new Error('connect ECONNREFUSED 127.0.0.1:2299'),
|
|
82
|
-
],
|
|
83
|
-
});
|
|
84
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.sshClientErrorMessage)(aggregate)).toBe('connect ECONNREFUSED ::1:2299; connect ECONNREFUSED 127.0.0.1:2299');
|
|
85
|
-
});
|
|
86
|
-
(0, vitest_1.it)('falls back to the code, then a fixed string', () => {
|
|
87
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.sshClientErrorMessage)(err('', { code: 'ETIMEDOUT' }))).toBe('ETIMEDOUT');
|
|
88
|
-
(0, vitest_1.expect)((0, sshTunnelFailure_1.sshClientErrorMessage)(err(''))).toBe('connection failed');
|
|
89
|
-
});
|
|
90
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"validateWarehouseColumnReferences.test.d.ts","sourceRoot":"","sources":["../src/validateWarehouseColumnReferences.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,168 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const common_1 = require("@lightdash/common");
|
|
4
|
-
const validateWarehouseColumnReferences_1 = require("./validateWarehouseColumnReferences");
|
|
5
|
-
const explore = {
|
|
6
|
-
name: 'orders',
|
|
7
|
-
label: 'Orders',
|
|
8
|
-
tags: [],
|
|
9
|
-
baseTable: 'orders',
|
|
10
|
-
joinedTables: [],
|
|
11
|
-
targetDatabase: common_1.SupportedDbtAdapter.POSTGRES,
|
|
12
|
-
tables: {
|
|
13
|
-
orders: {
|
|
14
|
-
name: 'orders',
|
|
15
|
-
label: 'Orders',
|
|
16
|
-
database: 'postgres',
|
|
17
|
-
schema: 'jaffle',
|
|
18
|
-
sqlTable: '"postgres"."jaffle"."orders"',
|
|
19
|
-
lineageGraph: {},
|
|
20
|
-
dimensions: {
|
|
21
|
-
amount: {
|
|
22
|
-
fieldType: common_1.FieldType.DIMENSION,
|
|
23
|
-
type: common_1.DimensionType.NUMBER,
|
|
24
|
-
name: 'amount',
|
|
25
|
-
label: 'Amount',
|
|
26
|
-
table: 'orders',
|
|
27
|
-
tableLabel: 'Orders',
|
|
28
|
-
// eslint-disable-next-line no-template-curly-in-string
|
|
29
|
-
sql: '${TABLE}.amount',
|
|
30
|
-
compiledSql: '"orders".amount',
|
|
31
|
-
tablesReferences: ['orders'],
|
|
32
|
-
hidden: false,
|
|
33
|
-
},
|
|
34
|
-
},
|
|
35
|
-
metrics: {
|
|
36
|
-
stale_metric: {
|
|
37
|
-
fieldType: common_1.FieldType.METRIC,
|
|
38
|
-
type: common_1.MetricType.AVERAGE,
|
|
39
|
-
name: 'stale_metric',
|
|
40
|
-
label: 'Stale metric',
|
|
41
|
-
table: 'orders',
|
|
42
|
-
tableLabel: 'Orders',
|
|
43
|
-
// eslint-disable-next-line no-template-curly-in-string
|
|
44
|
-
sql: '${TABLE}.rolling_30d_avg_sales',
|
|
45
|
-
compiledSql: 'AVG("orders".rolling_30d_avg_sales)',
|
|
46
|
-
tablesReferences: ['orders'],
|
|
47
|
-
hidden: false,
|
|
48
|
-
},
|
|
49
|
-
},
|
|
50
|
-
},
|
|
51
|
-
},
|
|
52
|
-
};
|
|
53
|
-
const queryResult = { fields: {}, rows: [] };
|
|
54
|
-
const tags = { query_context: 'cli' };
|
|
55
|
-
describe('validateWarehouseColumnReferences', () => {
|
|
56
|
-
it('returns explores unchanged when the warehouse accepts all references', async () => {
|
|
57
|
-
const runQuery = vi.fn(async () => queryResult);
|
|
58
|
-
const result = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
|
|
59
|
-
explores: [explore],
|
|
60
|
-
client: { getFieldQuoteChar: () => '"', runQuery },
|
|
61
|
-
tags,
|
|
62
|
-
});
|
|
63
|
-
expect(result).toEqual([explore]);
|
|
64
|
-
expect(runQuery).toHaveBeenCalledTimes(1);
|
|
65
|
-
expect(runQuery).toHaveBeenNthCalledWith(1, 'SELECT "orders".amount, "orders".rolling_30d_avg_sales FROM "postgres"."jaffle"."orders" AS "orders" WHERE 1 = 0', tags);
|
|
66
|
-
});
|
|
67
|
-
it('attaches one warning for each reference rejected by the warehouse', async () => {
|
|
68
|
-
const runQuery = vi.fn(async (sql) => {
|
|
69
|
-
if (sql.includes('rolling_30d_avg_sales')) {
|
|
70
|
-
throw new Error('column orders.rolling_30d_avg_sales does not exist');
|
|
71
|
-
}
|
|
72
|
-
return queryResult;
|
|
73
|
-
});
|
|
74
|
-
const [result] = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
|
|
75
|
-
explores: [explore],
|
|
76
|
-
client: { getFieldQuoteChar: () => '"', runQuery },
|
|
77
|
-
tags,
|
|
78
|
-
});
|
|
79
|
-
expect(result).toEqual({
|
|
80
|
-
...explore,
|
|
81
|
-
warnings: [
|
|
82
|
-
{
|
|
83
|
-
type: common_1.InlineErrorType.WAREHOUSE_COLUMN_ERROR,
|
|
84
|
-
message:
|
|
85
|
-
// eslint-disable-next-line no-template-curly-in-string
|
|
86
|
-
'Warehouse rejected column reference ${TABLE}.rolling_30d_avg_sales in model "orders": column orders.rolling_30d_avg_sales does not exist',
|
|
87
|
-
},
|
|
88
|
-
],
|
|
89
|
-
});
|
|
90
|
-
expect(runQuery).toHaveBeenCalledTimes(5);
|
|
91
|
-
});
|
|
92
|
-
it('skips column validation when the relation probe fails', async () => {
|
|
93
|
-
const runQuery = vi.fn(async () => {
|
|
94
|
-
throw new Error('warehouse unavailable');
|
|
95
|
-
});
|
|
96
|
-
const result = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
|
|
97
|
-
explores: [explore],
|
|
98
|
-
client: { getFieldQuoteChar: () => '"', runQuery },
|
|
99
|
-
tags,
|
|
100
|
-
});
|
|
101
|
-
expect(result).toEqual([explore]);
|
|
102
|
-
expect(runQuery).toHaveBeenCalledTimes(2);
|
|
103
|
-
});
|
|
104
|
-
it('preserves pre-existing warnings and appends warehouse warnings after them', async () => {
|
|
105
|
-
const existingWarning = {
|
|
106
|
-
type: common_1.InlineErrorType.FIELD_ERROR,
|
|
107
|
-
message: 'Dimension "broken_dimension" failed to compile',
|
|
108
|
-
};
|
|
109
|
-
const exploreWithExistingWarning = {
|
|
110
|
-
...explore,
|
|
111
|
-
warnings: [existingWarning],
|
|
112
|
-
};
|
|
113
|
-
const runQuery = vi.fn(async (sql) => {
|
|
114
|
-
if (sql.includes('rolling_30d_avg_sales')) {
|
|
115
|
-
throw new Error('column orders.rolling_30d_avg_sales does not exist');
|
|
116
|
-
}
|
|
117
|
-
return queryResult;
|
|
118
|
-
});
|
|
119
|
-
const [result] = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
|
|
120
|
-
explores: [exploreWithExistingWarning],
|
|
121
|
-
client: { getFieldQuoteChar: () => '"', runQuery },
|
|
122
|
-
tags,
|
|
123
|
-
});
|
|
124
|
-
expect(result).toEqual({
|
|
125
|
-
...exploreWithExistingWarning,
|
|
126
|
-
warnings: [
|
|
127
|
-
existingWarning,
|
|
128
|
-
{
|
|
129
|
-
type: common_1.InlineErrorType.WAREHOUSE_COLUMN_ERROR,
|
|
130
|
-
message:
|
|
131
|
-
// eslint-disable-next-line no-template-curly-in-string
|
|
132
|
-
'Warehouse rejected column reference ${TABLE}.rolling_30d_avg_sales in model "orders": column orders.rolling_30d_avg_sales does not exist',
|
|
133
|
-
},
|
|
134
|
-
],
|
|
135
|
-
});
|
|
136
|
-
});
|
|
137
|
-
it('skips explore errors and templated table references without probing', async () => {
|
|
138
|
-
const exploreError = {
|
|
139
|
-
name: 'broken_explore',
|
|
140
|
-
label: 'Broken explore',
|
|
141
|
-
errors: [
|
|
142
|
-
{
|
|
143
|
-
type: common_1.InlineErrorType.METADATA_PARSE_ERROR,
|
|
144
|
-
message: 'Failed to compile explore',
|
|
145
|
-
},
|
|
146
|
-
],
|
|
147
|
-
};
|
|
148
|
-
const templatedExplore = {
|
|
149
|
-
...explore,
|
|
150
|
-
name: 'templated_orders',
|
|
151
|
-
tables: {
|
|
152
|
-
orders: {
|
|
153
|
-
...explore.tables.orders,
|
|
154
|
-
// eslint-disable-next-line no-template-curly-in-string
|
|
155
|
-
sqlTable: '"postgres"."${ld.schema}"."orders"',
|
|
156
|
-
},
|
|
157
|
-
},
|
|
158
|
-
};
|
|
159
|
-
const runQuery = vi.fn(async () => queryResult);
|
|
160
|
-
const result = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
|
|
161
|
-
explores: [exploreError, templatedExplore],
|
|
162
|
-
client: { getFieldQuoteChar: () => '"', runQuery },
|
|
163
|
-
tags,
|
|
164
|
-
});
|
|
165
|
-
expect(result).toEqual([exploreError, templatedExplore]);
|
|
166
|
-
expect(runQuery).not.toHaveBeenCalled();
|
|
167
|
-
});
|
|
168
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"AthenaWarehouseClient.test.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/AthenaWarehouseClient.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,261 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
/* eslint-disable prefer-arrow-callback, func-names */
|
|
4
|
-
const common_1 = require("@lightdash/common");
|
|
5
|
-
const { mockAthenaClient } = vi.hoisted(() => ({
|
|
6
|
-
mockAthenaClient: vi.fn(),
|
|
7
|
-
}));
|
|
8
|
-
vi.mock('@aws-sdk/client-athena', async () => ({
|
|
9
|
-
...(await vi.importActual('@aws-sdk/client-athena')),
|
|
10
|
-
AthenaClient: mockAthenaClient,
|
|
11
|
-
}));
|
|
12
|
-
const { mockFromTemporaryCredentials } = vi.hoisted(() => ({
|
|
13
|
-
mockFromTemporaryCredentials: vi.fn(() => 'sts-credentials'),
|
|
14
|
-
}));
|
|
15
|
-
vi.mock('@aws-sdk/credential-providers', () => ({
|
|
16
|
-
fromTemporaryCredentials: mockFromTemporaryCredentials,
|
|
17
|
-
}));
|
|
18
|
-
// eslint-disable-next-line import/first -- Must import after mocks are set up
|
|
19
|
-
const AthenaWarehouseClient_1 = require("./AthenaWarehouseClient");
|
|
20
|
-
const baseCredentials = {
|
|
21
|
-
type: common_1.WarehouseTypes.ATHENA,
|
|
22
|
-
region: 'us-east-1',
|
|
23
|
-
database: 'AwsDataCatalog',
|
|
24
|
-
schema: 'my_database',
|
|
25
|
-
s3StagingDir: 's3://bucket/staging/',
|
|
26
|
-
authenticationType: common_1.AthenaAuthenticationType.ACCESS_KEY,
|
|
27
|
-
accessKeyId: 'AKID',
|
|
28
|
-
secretAccessKey: 'SECRET',
|
|
29
|
-
};
|
|
30
|
-
describe('convertDataTypeToDimensionType', () => {
|
|
31
|
-
test.each([
|
|
32
|
-
[AthenaWarehouseClient_1.AthenaTypes.FLOAT, common_1.DimensionType.NUMBER],
|
|
33
|
-
['FLOAT', common_1.DimensionType.NUMBER],
|
|
34
|
-
[AthenaWarehouseClient_1.AthenaTypes.DOUBLE, common_1.DimensionType.NUMBER],
|
|
35
|
-
[AthenaWarehouseClient_1.AthenaTypes.BIGINT, common_1.DimensionType.NUMBER],
|
|
36
|
-
['decimal(10, 2)', common_1.DimensionType.NUMBER],
|
|
37
|
-
])('maps Athena %s columns to %s dimensions', (input, expected) => {
|
|
38
|
-
expect((0, AthenaWarehouseClient_1.convertDataTypeToDimensionType)(input)).toBe(expected);
|
|
39
|
-
});
|
|
40
|
-
});
|
|
41
|
-
describe('AthenaWarehouseClient', () => {
|
|
42
|
-
beforeEach(() => {
|
|
43
|
-
vi.clearAllMocks();
|
|
44
|
-
mockAthenaClient.mockImplementation(function () {
|
|
45
|
-
return {};
|
|
46
|
-
});
|
|
47
|
-
});
|
|
48
|
-
describe('authentication', () => {
|
|
49
|
-
test('should use static credentials for ACCESS_KEY auth', () => {
|
|
50
|
-
// eslint-disable-next-line no-new
|
|
51
|
-
new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
52
|
-
expect(mockAthenaClient).toHaveBeenCalledWith({
|
|
53
|
-
region: 'us-east-1',
|
|
54
|
-
credentials: {
|
|
55
|
-
accessKeyId: 'AKID',
|
|
56
|
-
secretAccessKey: 'SECRET',
|
|
57
|
-
},
|
|
58
|
-
});
|
|
59
|
-
});
|
|
60
|
-
test('should forward sessionToken for temporary STS credentials', () => {
|
|
61
|
-
const creds = {
|
|
62
|
-
...baseCredentials,
|
|
63
|
-
accessKeyId: 'ASIATEST',
|
|
64
|
-
secretAccessKey: 'SECRET',
|
|
65
|
-
sessionToken: 'SESSIONTOKEN',
|
|
66
|
-
};
|
|
67
|
-
// eslint-disable-next-line no-new
|
|
68
|
-
new AthenaWarehouseClient_1.AthenaWarehouseClient(creds);
|
|
69
|
-
expect(mockAthenaClient).toHaveBeenCalledWith({
|
|
70
|
-
region: 'us-east-1',
|
|
71
|
-
credentials: {
|
|
72
|
-
accessKeyId: 'ASIATEST',
|
|
73
|
-
secretAccessKey: 'SECRET',
|
|
74
|
-
sessionToken: 'SESSIONTOKEN',
|
|
75
|
-
},
|
|
76
|
-
});
|
|
77
|
-
});
|
|
78
|
-
test('should not set credentials for IAM_ROLE auth', () => {
|
|
79
|
-
const creds = {
|
|
80
|
-
...baseCredentials,
|
|
81
|
-
authenticationType: common_1.AthenaAuthenticationType.IAM_ROLE,
|
|
82
|
-
};
|
|
83
|
-
// eslint-disable-next-line no-new
|
|
84
|
-
new AthenaWarehouseClient_1.AthenaWarehouseClient(creds);
|
|
85
|
-
expect(mockAthenaClient).toHaveBeenCalledWith({
|
|
86
|
-
region: 'us-east-1',
|
|
87
|
-
});
|
|
88
|
-
});
|
|
89
|
-
});
|
|
90
|
-
describe('assume role', () => {
|
|
91
|
-
test('should wrap credentials with fromTemporaryCredentials when assumeRoleArn is set', () => {
|
|
92
|
-
const creds = {
|
|
93
|
-
...baseCredentials,
|
|
94
|
-
assumeRoleArn: 'arn:aws:iam::123456789012:role/my-role',
|
|
95
|
-
assumeRoleExternalId: 'ext-id',
|
|
96
|
-
};
|
|
97
|
-
// eslint-disable-next-line no-new
|
|
98
|
-
new AthenaWarehouseClient_1.AthenaWarehouseClient(creds);
|
|
99
|
-
expect(mockFromTemporaryCredentials).toHaveBeenCalledWith({
|
|
100
|
-
masterCredentials: {
|
|
101
|
-
accessKeyId: 'AKID',
|
|
102
|
-
secretAccessKey: 'SECRET',
|
|
103
|
-
},
|
|
104
|
-
params: {
|
|
105
|
-
RoleArn: 'arn:aws:iam::123456789012:role/my-role',
|
|
106
|
-
RoleSessionName: 'lightdash-athena-session',
|
|
107
|
-
ExternalId: 'ext-id',
|
|
108
|
-
},
|
|
109
|
-
});
|
|
110
|
-
expect(mockAthenaClient).toHaveBeenCalledWith({
|
|
111
|
-
region: 'us-east-1',
|
|
112
|
-
credentials: 'sts-credentials',
|
|
113
|
-
});
|
|
114
|
-
});
|
|
115
|
-
test('should chain assume role using temporary credentials (sessionToken in masterCredentials)', () => {
|
|
116
|
-
const creds = {
|
|
117
|
-
...baseCredentials,
|
|
118
|
-
accessKeyId: 'ASIATEST',
|
|
119
|
-
secretAccessKey: 'SECRET',
|
|
120
|
-
sessionToken: 'SESSIONTOKEN',
|
|
121
|
-
assumeRoleArn: 'arn:aws:iam::123456789012:role/my-role',
|
|
122
|
-
};
|
|
123
|
-
// eslint-disable-next-line no-new
|
|
124
|
-
new AthenaWarehouseClient_1.AthenaWarehouseClient(creds);
|
|
125
|
-
expect(mockFromTemporaryCredentials).toHaveBeenCalledWith({
|
|
126
|
-
masterCredentials: {
|
|
127
|
-
accessKeyId: 'ASIATEST',
|
|
128
|
-
secretAccessKey: 'SECRET',
|
|
129
|
-
sessionToken: 'SESSIONTOKEN',
|
|
130
|
-
},
|
|
131
|
-
params: {
|
|
132
|
-
RoleArn: 'arn:aws:iam::123456789012:role/my-role',
|
|
133
|
-
RoleSessionName: 'lightdash-athena-session',
|
|
134
|
-
ExternalId: undefined,
|
|
135
|
-
},
|
|
136
|
-
});
|
|
137
|
-
});
|
|
138
|
-
test('should not wrap credentials when assumeRoleArn is not set', () => {
|
|
139
|
-
// eslint-disable-next-line no-new
|
|
140
|
-
new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
141
|
-
expect(mockFromTemporaryCredentials).not.toHaveBeenCalled();
|
|
142
|
-
});
|
|
143
|
-
test('should work with IAM_ROLE and assume role together', () => {
|
|
144
|
-
const creds = {
|
|
145
|
-
...baseCredentials,
|
|
146
|
-
authenticationType: common_1.AthenaAuthenticationType.IAM_ROLE,
|
|
147
|
-
assumeRoleArn: 'arn:aws:iam::123456789012:role/my-role',
|
|
148
|
-
};
|
|
149
|
-
// eslint-disable-next-line no-new
|
|
150
|
-
new AthenaWarehouseClient_1.AthenaWarehouseClient(creds);
|
|
151
|
-
expect(mockFromTemporaryCredentials).toHaveBeenCalledWith({
|
|
152
|
-
masterCredentials: undefined,
|
|
153
|
-
params: {
|
|
154
|
-
RoleArn: 'arn:aws:iam::123456789012:role/my-role',
|
|
155
|
-
RoleSessionName: 'lightdash-athena-session',
|
|
156
|
-
ExternalId: undefined,
|
|
157
|
-
},
|
|
158
|
-
});
|
|
159
|
-
expect(mockAthenaClient).toHaveBeenCalledWith({
|
|
160
|
-
region: 'us-east-1',
|
|
161
|
-
credentials: 'sts-credentials',
|
|
162
|
-
});
|
|
163
|
-
});
|
|
164
|
-
});
|
|
165
|
-
describe('error translation', () => {
|
|
166
|
-
// Synthesizes an error with the shape produced by the AWS SDK:
|
|
167
|
-
// an Error subclass whose `name` is the AWS error code, with optional
|
|
168
|
-
// $metadata.httpStatusCode (set by the SDK on every ServiceException).
|
|
169
|
-
const makeAwsError = (name, message, httpStatusCode) => {
|
|
170
|
-
const err = new Error(message);
|
|
171
|
-
err.name = name;
|
|
172
|
-
if (httpStatusCode !== undefined) {
|
|
173
|
-
err.$metadata = { httpStatusCode };
|
|
174
|
-
}
|
|
175
|
-
return err;
|
|
176
|
-
};
|
|
177
|
-
const setMockSendToReject = (error) => {
|
|
178
|
-
mockAthenaClient.mockImplementation(function () {
|
|
179
|
-
return { send: vi.fn().mockRejectedValue(error) };
|
|
180
|
-
});
|
|
181
|
-
};
|
|
182
|
-
test('translates UnrecognizedClientException into WarehouseConnectionError with hint', async () => {
|
|
183
|
-
setMockSendToReject(makeAwsError('UnrecognizedClientException', 'The security token included in the request is invalid.'));
|
|
184
|
-
const client = new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
185
|
-
await expect(client.test()).rejects.toBeInstanceOf(common_1.WarehouseConnectionError);
|
|
186
|
-
await expect(client.test()).rejects.toMatchObject({
|
|
187
|
-
message: expect.stringContaining('[UnrecognizedClientException]'),
|
|
188
|
-
});
|
|
189
|
-
await expect(client.test()).rejects.toMatchObject({
|
|
190
|
-
message: expect.stringContaining('AWS rejected the access key ID'),
|
|
191
|
-
});
|
|
192
|
-
});
|
|
193
|
-
test('translates InvalidSignatureException into WarehouseConnectionError', async () => {
|
|
194
|
-
setMockSendToReject(makeAwsError('InvalidSignatureException', 'Signature does not match.'));
|
|
195
|
-
const client = new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
196
|
-
await expect(client.test()).rejects.toBeInstanceOf(common_1.WarehouseConnectionError);
|
|
197
|
-
await expect(client.test()).rejects.toMatchObject({
|
|
198
|
-
message: expect.stringContaining('secret access key'),
|
|
199
|
-
});
|
|
200
|
-
});
|
|
201
|
-
test('translates ExpiredTokenException into WarehouseConnectionError', async () => {
|
|
202
|
-
setMockSendToReject(makeAwsError('ExpiredTokenException', 'The security token has expired.'));
|
|
203
|
-
const client = new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
204
|
-
await expect(client.test()).rejects.toBeInstanceOf(common_1.WarehouseConnectionError);
|
|
205
|
-
await expect(client.test()).rejects.toMatchObject({
|
|
206
|
-
message: expect.stringContaining('expired'),
|
|
207
|
-
});
|
|
208
|
-
});
|
|
209
|
-
test('translates CredentialsProviderError into WarehouseConnectionError', async () => {
|
|
210
|
-
setMockSendToReject(makeAwsError('CredentialsProviderError', 'Could not load credentials from any providers'));
|
|
211
|
-
const client = new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
212
|
-
await expect(client.test()).rejects.toBeInstanceOf(common_1.WarehouseConnectionError);
|
|
213
|
-
await expect(client.test()).rejects.toMatchObject({
|
|
214
|
-
message: expect.stringContaining('IAM Role'),
|
|
215
|
-
});
|
|
216
|
-
});
|
|
217
|
-
test('keeps non-auth errors as WarehouseQueryError from streamQuery', async () => {
|
|
218
|
-
setMockSendToReject(makeAwsError('InvalidRequestException', 'Workgroup primary not found'));
|
|
219
|
-
const client = new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
220
|
-
await expect(client.test()).rejects.toBeInstanceOf(common_1.WarehouseQueryError);
|
|
221
|
-
await expect(client.test()).rejects.toMatchObject({
|
|
222
|
-
message: expect.stringContaining('[InvalidRequestException]'),
|
|
223
|
-
});
|
|
224
|
-
});
|
|
225
|
-
test('falls back to httpStatusCode=401 when error name is unfamiliar', async () => {
|
|
226
|
-
// Some AWS error variants don't show up in our known-name set but
|
|
227
|
-
// are still authentication failures (e.g. credential-provider chain
|
|
228
|
-
// wrapping). HTTP 401 alone should be enough to classify as a
|
|
229
|
-
// connection error.
|
|
230
|
-
setMockSendToReject(makeAwsError('SomeFutureAwsAuthError', 'unauthenticated', 401));
|
|
231
|
-
const client = new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
232
|
-
await expect(client.test()).rejects.toBeInstanceOf(common_1.WarehouseConnectionError);
|
|
233
|
-
await expect(client.test()).rejects.toMatchObject({
|
|
234
|
-
message: expect.stringContaining('[SomeFutureAwsAuthError 401]'),
|
|
235
|
-
});
|
|
236
|
-
});
|
|
237
|
-
test('does NOT promote httpStatusCode=403 to a connection error (could be IAM gap)', async () => {
|
|
238
|
-
// AccessDeniedException is 403 but represents an IAM permission
|
|
239
|
-
// gap during a query — it should remain WarehouseQueryError when
|
|
240
|
-
// streamQuery is the caller. The catalog/tables/fields paths
|
|
241
|
-
// explicitly opt into 'connection' default elsewhere.
|
|
242
|
-
setMockSendToReject(makeAwsError('AccessDeniedException', 'You are not authorized to perform: athena:GetQueryResults', 403));
|
|
243
|
-
const client = new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
244
|
-
await expect(client.test()).rejects.toBeInstanceOf(common_1.WarehouseQueryError);
|
|
245
|
-
await expect(client.test()).rejects.toMatchObject({
|
|
246
|
-
message: expect.stringContaining('[AccessDeniedException 403]'),
|
|
247
|
-
});
|
|
248
|
-
});
|
|
249
|
-
test('catalog/tables/fields default to WarehouseConnectionError', async () => {
|
|
250
|
-
setMockSendToReject(makeAwsError('AccessDeniedException', 'You are not authorized to perform: athena:ListTableMetadata'));
|
|
251
|
-
const client = new AthenaWarehouseClient_1.AthenaWarehouseClient(baseCredentials);
|
|
252
|
-
await expect(client.getAllTables()).rejects.toBeInstanceOf(common_1.WarehouseConnectionError);
|
|
253
|
-
await expect(client.getAllTables()).rejects.toMatchObject({
|
|
254
|
-
message: expect.stringContaining('[AccessDeniedException]'),
|
|
255
|
-
});
|
|
256
|
-
await expect(client.getAllTables()).rejects.toMatchObject({
|
|
257
|
-
message: expect.stringContaining('Failed to list tables'),
|
|
258
|
-
});
|
|
259
|
-
});
|
|
260
|
-
});
|
|
261
|
-
});
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
import { AnyType, CreateBigqueryCredentials } from '@lightdash/common';
|
|
2
|
-
import { Readable } from 'stream';
|
|
3
|
-
import { BigqueryFieldType } from './BigqueryWarehouseClient';
|
|
4
|
-
export declare const credentials: CreateBigqueryCredentials;
|
|
5
|
-
export declare const getTableResponse: {
|
|
6
|
-
id: string;
|
|
7
|
-
bigQuery: {
|
|
8
|
-
projectId: string;
|
|
9
|
-
};
|
|
10
|
-
getMetadata: import("vitest").Mock<() => {
|
|
11
|
-
schema: {
|
|
12
|
-
fields: ({
|
|
13
|
-
name: string;
|
|
14
|
-
type: BigqueryFieldType;
|
|
15
|
-
mode?: undefined;
|
|
16
|
-
fields?: undefined;
|
|
17
|
-
} | {
|
|
18
|
-
name: string;
|
|
19
|
-
type: BigqueryFieldType;
|
|
20
|
-
mode: string;
|
|
21
|
-
fields?: undefined;
|
|
22
|
-
} | {
|
|
23
|
-
name: string;
|
|
24
|
-
type: BigqueryFieldType;
|
|
25
|
-
mode: string;
|
|
26
|
-
fields: ({
|
|
27
|
-
mode?: undefined;
|
|
28
|
-
fields?: undefined;
|
|
29
|
-
name: string;
|
|
30
|
-
type: BigqueryFieldType;
|
|
31
|
-
} | {
|
|
32
|
-
name: string;
|
|
33
|
-
type: BigqueryFieldType;
|
|
34
|
-
mode: string;
|
|
35
|
-
fields: {
|
|
36
|
-
name: string;
|
|
37
|
-
type: BigqueryFieldType;
|
|
38
|
-
}[];
|
|
39
|
-
})[];
|
|
40
|
-
})[];
|
|
41
|
-
};
|
|
42
|
-
}[]>;
|
|
43
|
-
};
|
|
44
|
-
export declare const rows: Record<string, AnyType>[];
|
|
45
|
-
declare const mockStreamRow: () => Readable;
|
|
46
|
-
export declare const createJobResponse: {
|
|
47
|
-
getQueryResults: import("vitest").Mock<() => (Record<string, any>[] | {
|
|
48
|
-
schema: {
|
|
49
|
-
fields: ({
|
|
50
|
-
name: string;
|
|
51
|
-
type: BigqueryFieldType;
|
|
52
|
-
mode?: undefined;
|
|
53
|
-
fields?: undefined;
|
|
54
|
-
} | {
|
|
55
|
-
name: string;
|
|
56
|
-
type: BigqueryFieldType;
|
|
57
|
-
mode: string;
|
|
58
|
-
fields?: undefined;
|
|
59
|
-
} | {
|
|
60
|
-
name: string;
|
|
61
|
-
type: BigqueryFieldType;
|
|
62
|
-
mode: string;
|
|
63
|
-
fields: ({
|
|
64
|
-
mode?: undefined;
|
|
65
|
-
fields?: undefined;
|
|
66
|
-
name: string;
|
|
67
|
-
type: BigqueryFieldType;
|
|
68
|
-
} | {
|
|
69
|
-
name: string;
|
|
70
|
-
type: BigqueryFieldType;
|
|
71
|
-
mode: string;
|
|
72
|
-
fields: {
|
|
73
|
-
name: string;
|
|
74
|
-
type: BigqueryFieldType;
|
|
75
|
-
}[];
|
|
76
|
-
})[];
|
|
77
|
-
})[];
|
|
78
|
-
};
|
|
79
|
-
} | undefined)[]>;
|
|
80
|
-
getQueryResultsStream: import("vitest").Mock<typeof mockStreamRow>;
|
|
81
|
-
}[];
|
|
82
|
-
export {};
|
|
83
|
-
//# sourceMappingURL=BigqueryWarehouseClient.mock.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"BigqueryWarehouseClient.mock.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.mock.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,yBAAyB,EAE5B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,eAAO,MAAM,WAAW,EAAE,yBAUzB,CAAC;AA+DF,eAAO,MAAM,gBAAgB;IACzB,EAAE;IACF,QAAQ;QAAI,SAAS;;IACrB,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CACd,CAAC;AAwBF,eAAO,MAAM,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAgBzC,CAAC;AAEF,QAAA,MAAM,aAAa,gBAOb,CAAC;AAEP,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAK7B,CAAC"}
|