@cubejs-backend/redshift-driver 0.30.4 → 0.30.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,30 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
5
|
|
|
6
|
+
## [0.30.7](https://github.com/cube-js/cube.js/compare/v0.30.6...v0.30.7) (2022-05-26)
|
|
7
|
+
|
|
8
|
+
**Note:** Version bump only for package @cubejs-backend/redshift-driver
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
## [0.30.6](https://github.com/cube-js/cube.js/compare/v0.30.5...v0.30.6) (2022-05-24)
|
|
15
|
+
|
|
16
|
+
**Note:** Version bump only for package @cubejs-backend/redshift-driver
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
## [0.30.5](https://github.com/cube-js/cube.js/compare/v0.30.4...v0.30.5) (2022-05-23)
|
|
23
|
+
|
|
24
|
+
**Note:** Version bump only for package @cubejs-backend/redshift-driver
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
6
30
|
## [0.30.4](https://github.com/cube-js/cube.js/compare/v0.30.3...v0.30.4) (2022-05-20)
|
|
7
31
|
|
|
8
32
|
**Note:** Version bump only for package @cubejs-backend/redshift-driver
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
import { PostgresDriver, PostgresDriverConfiguration } from '@cubejs-backend/postgres-driver';
|
|
2
2
|
import { DownloadTableCSVData, UnloadOptions } from '@cubejs-backend/query-orchestrator';
|
|
3
|
-
interface
|
|
3
|
+
interface RedshiftDriverExportRequiredAWS {
|
|
4
4
|
bucketType: 's3';
|
|
5
5
|
bucketName: string;
|
|
6
|
-
keyId: string;
|
|
7
|
-
secretKey: string;
|
|
8
6
|
region: string;
|
|
9
7
|
}
|
|
8
|
+
interface RedshiftDriverExportArnAWS extends RedshiftDriverExportRequiredAWS {
|
|
9
|
+
unloadArn?: string;
|
|
10
|
+
}
|
|
11
|
+
interface RedshiftDriverExportKeySecretAWS extends RedshiftDriverExportRequiredAWS {
|
|
12
|
+
keyId?: string;
|
|
13
|
+
secretKey?: string;
|
|
14
|
+
}
|
|
15
|
+
interface RedshiftDriverExportAWS extends RedshiftDriverExportArnAWS, RedshiftDriverExportKeySecretAWS {
|
|
16
|
+
}
|
|
10
17
|
export interface RedshiftDriverConfiguration extends PostgresDriverConfiguration {
|
|
11
18
|
exportBucket?: RedshiftDriverExportAWS;
|
|
12
19
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RedshiftDriver.d.ts","sourceRoot":"","sources":["../../src/RedshiftDriver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,2BAA2B,EAAE,MAAM,iCAAiC,CAAC;AAC9F,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AAMzF,UAAU
|
|
1
|
+
{"version":3,"file":"RedshiftDriver.d.ts","sourceRoot":"","sources":["../../src/RedshiftDriver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,2BAA2B,EAAE,MAAM,iCAAiC,CAAC;AAC9F,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AAMzF,UAAU,+BAA+B;IACvC,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,0BAA2B,SAAQ,+BAA+B;IAI1E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,gCAAiC,SAAQ,+BAA+B;IAChF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,uBAAwB,SAAQ,0BAA0B,EAAE,gCAAgC;CACrG;AAED,MAAM,WAAW,2BAA4B,SAAQ,2BAA2B;IAC9E,YAAY,CAAC,EAAE,uBAAuB,CAAC;CACxC;AAED,qBAAa,cAAe,SAAQ,cAAc,CAAC,2BAA2B,CAAC;gBAC1D,OAAO,GAAE,2BAAgC;IAI5D,SAAS,CAAC,uBAAuB,IAAI,OAAO,CAAC,2BAA2B,CAAC;IAQzE,SAAS,CAAC,eAAe,IAAI,uBAAuB,GAAG,SAAS;IAoDnD,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrC,iBAAiB;IAQjB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,oBAAoB,CAAC;CAmG1F"}
|
|
@@ -21,24 +21,39 @@ class RedshiftDriver extends postgres_driver_1.PostgresDriver {
|
|
|
21
21
|
};
|
|
22
22
|
}
|
|
23
23
|
getExportBucket() {
|
|
24
|
-
const
|
|
24
|
+
const supportedBucketTypes = ['s3'];
|
|
25
|
+
const requiredExportBucket = {
|
|
25
26
|
bucketType: shared_1.getEnv('dbExportBucketType', {
|
|
26
|
-
supported:
|
|
27
|
+
supported: supportedBucketTypes
|
|
27
28
|
}),
|
|
28
29
|
bucketName: shared_1.getEnv('dbExportBucket'),
|
|
30
|
+
region: shared_1.getEnv('dbExportBucketAwsRegion'),
|
|
31
|
+
};
|
|
32
|
+
const exportBucket = {
|
|
33
|
+
...requiredExportBucket,
|
|
29
34
|
keyId: shared_1.getEnv('dbExportBucketAwsKey'),
|
|
30
35
|
secretKey: shared_1.getEnv('dbExportBucketAwsSecret'),
|
|
31
|
-
|
|
36
|
+
unloadArn: shared_1.getEnv('dbExportBucketRedshiftArn')
|
|
32
37
|
};
|
|
33
38
|
if (exportBucket.bucketType) {
|
|
34
|
-
const supportedBucketTypes = ['s3'];
|
|
35
39
|
if (!supportedBucketTypes.includes(exportBucket.bucketType)) {
|
|
36
40
|
throw new Error(`Unsupported EXPORT_BUCKET_TYPE, supported: ${supportedBucketTypes.join(',')}`);
|
|
37
41
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
+
// Make sure the required keys are set
|
|
43
|
+
const emptyRequiredKeys = Object.keys(requiredExportBucket)
|
|
44
|
+
.filter((key) => requiredExportBucket[key] === undefined);
|
|
45
|
+
if (emptyRequiredKeys.length) {
|
|
46
|
+
throw new Error(`Unsupported configuration exportBucket, some configuration keys are empty: ${emptyRequiredKeys.join(',')}`);
|
|
47
|
+
}
|
|
48
|
+
// If unload ARN is not set, secret and key id must be set for Redshift
|
|
49
|
+
if (!exportBucket.unloadArn) {
|
|
50
|
+
// Make sure the required keys are set
|
|
51
|
+
const emptySecretKeys = Object.keys(exportBucket)
|
|
52
|
+
.filter((key) => key !== 'unloadArn')
|
|
53
|
+
.filter((key) => exportBucket[key] === undefined);
|
|
54
|
+
if (emptySecretKeys.length) {
|
|
55
|
+
throw new Error(`Unsupported configuration exportBucket, some configuration keys are empty: ${emptySecretKeys.join(',')}`);
|
|
56
|
+
}
|
|
42
57
|
}
|
|
43
58
|
return exportBucket;
|
|
44
59
|
}
|
|
@@ -57,7 +72,7 @@ class RedshiftDriver extends postgres_driver_1.PostgresDriver {
|
|
|
57
72
|
if (!this.config.exportBucket) {
|
|
58
73
|
throw new Error('Unload is not configured');
|
|
59
74
|
}
|
|
60
|
-
const { bucketType, bucketName, keyId, secretKey
|
|
75
|
+
const { bucketType, bucketName, region, unloadArn, keyId, secretKey } = this.config.exportBucket;
|
|
61
76
|
const conn = await this.pool.connect();
|
|
62
77
|
try {
|
|
63
78
|
const exportPathName = crypto_1.default.randomBytes(10).toString('hex');
|
|
@@ -92,10 +107,15 @@ class RedshiftDriver extends postgres_driver_1.PostgresDriver {
|
|
|
92
107
|
}
|
|
93
108
|
}
|
|
94
109
|
});
|
|
110
|
+
const baseQuery = `UNLOAD ('SELECT * FROM ${table}') TO '${bucketType}://${bucketName}/${exportPathName}/'`;
|
|
111
|
+
// Prefer the unloadArn if it is present
|
|
112
|
+
const credentialQuery = unloadArn
|
|
113
|
+
? `iam_role '${unloadArn}'`
|
|
114
|
+
: `CREDENTIALS 'aws_access_key_id=${keyId};aws_secret_access_key=${secretKey}'`;
|
|
115
|
+
const unloadQuery = `${baseQuery} ${credentialQuery} ${optionsPart}`;
|
|
95
116
|
// Unable to extract number of extracted rows, because it's done in protocol notice
|
|
96
117
|
await conn.query({
|
|
97
|
-
text:
|
|
98
|
-
`CREDENTIALS 'aws_access_key_id=${keyId};aws_secret_access_key=${secretKey}' ${optionsPart}`),
|
|
118
|
+
text: unloadQuery,
|
|
99
119
|
});
|
|
100
120
|
if (unloadTotalRows === 0) {
|
|
101
121
|
return {
|
|
@@ -103,10 +123,10 @@ class RedshiftDriver extends postgres_driver_1.PostgresDriver {
|
|
|
103
123
|
};
|
|
104
124
|
}
|
|
105
125
|
const client = new client_s3_1.S3({
|
|
106
|
-
credentials: {
|
|
126
|
+
credentials: (keyId && secretKey) ? {
|
|
107
127
|
accessKeyId: keyId,
|
|
108
128
|
secretAccessKey: secretKey,
|
|
109
|
-
},
|
|
129
|
+
} : undefined,
|
|
110
130
|
region,
|
|
111
131
|
});
|
|
112
132
|
const list = await client.listObjectsV2({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RedshiftDriver.js","sourceRoot":"","sources":["../../src/RedshiftDriver.ts"],"names":[],"mappings":";;;;;;AAAA,qEAA8F;AAE9F,mDAAgD;AAChD,oDAA4B;AAC5B,wEAA6D;AAC7D,kDAA0D;
|
|
1
|
+
{"version":3,"file":"RedshiftDriver.js","sourceRoot":"","sources":["../../src/RedshiftDriver.ts"],"names":[],"mappings":";;;;;;AAAA,qEAA8F;AAE9F,mDAAgD;AAChD,oDAA4B;AAC5B,wEAA6D;AAC7D,kDAA0D;AA2B1D,MAAa,cAAe,SAAQ,gCAA2C;IAC7E,YAAmB,UAAuC,EAAE;QAC1D,KAAK,CAAC,OAAO,CAAC,CAAC;IACjB,CAAC;IAES,uBAAuB;QAC/B,OAAO;YACL,2GAA2G;YAC3G,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,IAAI,CAAC,eAAe,EAAE;SACrC,CAAC;IACJ,CAAC;IAES,eAAe;QACvB,MAAM,oBAAoB,GAAG,CAAC,IAAI,CAAC,CAAC;QAEpC,MAAM,oBAAoB,GAA6C;YACrE,UAAU,EAAE,eAAM,CAAC,oBAAoB,EAAE;gBACvC,SAAS,EAAE,oBAAoB;aAChC,CAAC;YACF,UAAU,EAAE,eAAM,CAAC,gBAAgB,CAAC;YACpC,MAAM,EAAE,eAAM,CAAC,yBAAyB,CAAC;SAC1C,CAAC;QAEF,MAAM,YAAY,GAAqC;YACrD,GAAG,oBAAoB;YACvB,KAAK,EAAE,eAAM,CAAC,sBAAsB,CAAC;YACrC,SAAS,EAAE,eAAM,CAAC,yBAAyB,CAAC;YAC5C,SAAS,EAAE,eAAM,CAAC,2BAA2B,CAAC;SAC/C,CAAC;QAEF,IAAI,YAAY,CAAC,UAAU,EAAE;YAC3B,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE;gBAC3D,MAAM,IAAI,KAAK,CACb,8CAA8C,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAC/E,CAAC;aACH;YAED,sCAAsC;YACtC,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC;iBACxD,MAAM,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,oBAAoB,CAAwC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC;YAC3G,IAAI,iBAAiB,CAAC,MAAM,EAAE;gBAC5B,MAAM,IAAI,KAAK,CACb,8EAA8E,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAC5G,CAAC;aACH;YACD,uEAAuE;YACvE,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE;gBAC3B,sCAAsC;gBACtC,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;qBAC9C,MAAM,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,KAAK,WAAW,CAAC;qBAC5C,MAAM,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,YAAY,CAAgC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC;gBAC3F,IAAI,eAAe,CAAC,MAAM,EAAE;oBAC1B,MAAM,IAAI,KAAK,CACb,8EAA8E,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAC1G,CAAC;iBACH;aACF;YAED,OAAgC,YAAY,CAAC;SAC9C;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEM,KAAK,CAAC,oBAAoB;QAC/B,iFAAiF;IACnF,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC5B,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;YAC5B,OAAO,IAAI,CAAC;SACb;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAsB;QACvD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;SAC7C;QAED,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QAEjG,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAEvC,IAAI;YACF,MAAM,cAAc,GAAG,gBAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAE9D,MAAM,eAAe,GAAG;gBACtB,MAAM,EAAE,IAAI,MAAM,GAAG;gBACrB,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,EAAE;gBACR,WAAW,EAAE,GAAG,OAAO,CAAC,WAAW,IAAI;aACxC,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC;iBAChD,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;iBACxC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEb,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE;gBACjC,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM;aAC9F,CAAC,CAAC;YAEH,IAAI,eAAe,GAAkB,IAAI,CAAC;YAE1C;;;;;eAKG;YACH,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAM,EAAE,EAAE;gBACpC,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE;oBACzD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBACvC,IAAI,OAAO,EAAE;wBACX,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;qBAC5C;yBAAM;wBACL,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;qBAChE;iBACF;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,0BAA0B,KAAK,UAAU,UAAU,MAAM,UAAU,IAAI,cAAc,IAAI,CAAC;YAC5G,wCAAwC;YACxC,MAAM,eAAe,GAAG,SAAS;gBAC/B,CAAC,CAAC,aAAa,SAAS,GAAG;gBAC3B,CAAC,CAAC,kCAAkC,KAAK,0BAA0B,SAAS,GAAG,CAAC;YAElF,MAAM,WAAW,GAAG,GAAG,SAAS,IAAI,eAAe,IAAI,WAAW,EAAE,CAAC;YAErE,mFAAmF;YACnF,MAAM,IAAI,CAAC,KAAK,CAAC;gBACf,IAAI,EAAE,WAAW;aAClB,CAAC,CAAC;YAEH,IAAI,eAAe,KAAK,CAAC,EAAE;gBACzB,OAAO;oBACL,OAAO,EAAE,EAAE;iBACZ,CAAC;aACH;YAED,MAAM,MAAM,GAAG,IAAI,cAAE,CAAC;gBACpB,WAAW,EAAE,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;oBAClC,WAAW,EAAE,KAAK;oBAClB,eAAe,EAAE,SAAS;iBAC3B,CAAC,CAAC,CAAC,SAAS;gBACb,MAAM;aACP,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;gBACtC,MAAM,EAAE,UAAU;gBAClB,MAAM,EAAE,cAAc;aACvB,CAAC,CAAC;YACH,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACzB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC/B,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC;wBACnC,MAAM,EAAE,UAAU;wBAClB,GAAG,EAAE,IAAI,CAAC,GAAG;qBACd,CAAC,CAAC;oBACH,OAAO,mCAAY,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5D,CAAC,CAAC,CACH,CAAC;gBAEF,OAAO;oBACL,OAAO;iBACR,CAAC;aACH;YAED,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;SAC7E;gBAAS;YACR,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YAElC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;SACtB;IACH,CAAC;CACF;AAhLD,wCAgLC"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@cubejs-backend/redshift-driver",
|
|
3
3
|
"description": "Cube.js Redshift database driver",
|
|
4
4
|
"author": "Cube Dev, Inc.",
|
|
5
|
-
"version": "0.30.
|
|
5
|
+
"version": "0.30.7",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "https://github.com/cube-js/cube.js.git",
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"@aws-sdk/client-s3": "^3.17.0",
|
|
29
29
|
"@aws-sdk/s3-request-presigner": "^3.17.0",
|
|
30
|
-
"@cubejs-backend/postgres-driver": "^0.30.
|
|
31
|
-
"@cubejs-backend/query-orchestrator": "^0.30.
|
|
32
|
-
"@cubejs-backend/shared": "^0.30.
|
|
30
|
+
"@cubejs-backend/postgres-driver": "^0.30.7",
|
|
31
|
+
"@cubejs-backend/query-orchestrator": "^0.30.7",
|
|
32
|
+
"@cubejs-backend/shared": "^0.30.7"
|
|
33
33
|
},
|
|
34
34
|
"license": "Apache-2.0",
|
|
35
35
|
"devDependencies": {
|
|
@@ -42,5 +42,5 @@
|
|
|
42
42
|
"eslintConfig": {
|
|
43
43
|
"extends": "../cubejs-linter"
|
|
44
44
|
},
|
|
45
|
-
"gitHead": "
|
|
45
|
+
"gitHead": "38c2b2e2d3331bb1549963618b17363f7b134448"
|
|
46
46
|
}
|