@forzalabs/remora 0.0.13 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Constants.js +1 -1
- package/definitions/json_schemas/source-schema.json +59 -8
- package/documentation/default_resources/consumer.json +3 -3
- package/drivers/RedshiftDriver.js +0 -2
- package/drivers/S3Driver.js +1 -1
- package/drivers/S3SourceDriver.js +1 -1
- package/engines/file/FileExporter.js +2 -1
- package/package.json +1 -1
package/Constants.js
CHANGED
|
@@ -41,14 +41,65 @@
|
|
|
41
41
|
"implicit"
|
|
42
42
|
],
|
|
43
43
|
"description": "The authentication method to use"
|
|
44
|
+
},
|
|
45
|
+
"host": {
|
|
46
|
+
"type": "string",
|
|
47
|
+
"description": "Hostname or endpoint of the data source"
|
|
48
|
+
},
|
|
49
|
+
"user": {
|
|
50
|
+
"type": "string",
|
|
51
|
+
"description": "Username for authentication"
|
|
52
|
+
},
|
|
53
|
+
"password": {
|
|
54
|
+
"type": "string",
|
|
55
|
+
"description": "Password for authentication"
|
|
56
|
+
},
|
|
57
|
+
"database": {
|
|
58
|
+
"type": "string",
|
|
59
|
+
"description": "Database name"
|
|
60
|
+
},
|
|
61
|
+
"workgroup": {
|
|
62
|
+
"type": "string",
|
|
63
|
+
"description": "Workgroup name"
|
|
64
|
+
},
|
|
65
|
+
"schema": {
|
|
66
|
+
"type": "string",
|
|
67
|
+
"description": "Database schema name"
|
|
68
|
+
},
|
|
69
|
+
"port": {
|
|
70
|
+
"type": "string",
|
|
71
|
+
"description": "Port number for the connection"
|
|
72
|
+
},
|
|
73
|
+
"accessKey": {
|
|
74
|
+
"type": "string",
|
|
75
|
+
"description": "AWS access key ID"
|
|
76
|
+
},
|
|
77
|
+
"secretKey": {
|
|
78
|
+
"type": "string",
|
|
79
|
+
"description": "AWS secret access key"
|
|
80
|
+
},
|
|
81
|
+
"sessionToken": {
|
|
82
|
+
"type": "string",
|
|
83
|
+
"description": "AWS session token (if required)"
|
|
84
|
+
},
|
|
85
|
+
"region": {
|
|
86
|
+
"type": "string",
|
|
87
|
+
"description": "AWS region"
|
|
88
|
+
},
|
|
89
|
+
"bucket": {
|
|
90
|
+
"type": "string",
|
|
91
|
+
"description": "S3 bucket name"
|
|
92
|
+
},
|
|
93
|
+
"iamProfile": {
|
|
94
|
+
"type": "string",
|
|
95
|
+
"description": "IAM role or profile name"
|
|
96
|
+
},
|
|
97
|
+
"clusterId": {
|
|
98
|
+
"type": "string",
|
|
99
|
+
"description": "Redshift cluster identifier"
|
|
44
100
|
}
|
|
45
101
|
},
|
|
46
|
-
"required": [
|
|
47
|
-
"method"
|
|
48
|
-
],
|
|
49
|
-
"additionalProperties": {
|
|
50
|
-
"type": "string"
|
|
51
|
-
}
|
|
102
|
+
"required": ["method"]
|
|
52
103
|
},
|
|
53
104
|
"_version": {
|
|
54
105
|
"type": "number",
|
|
@@ -72,7 +123,7 @@
|
|
|
72
123
|
"host": "prod-db.example.com",
|
|
73
124
|
"user": "app_user",
|
|
74
125
|
"password": "password123",
|
|
75
|
-
"
|
|
126
|
+
"database": "customers",
|
|
76
127
|
"schema": "public",
|
|
77
128
|
"port": "5432"
|
|
78
129
|
},
|
|
@@ -97,7 +148,7 @@
|
|
|
97
148
|
"authentication": {
|
|
98
149
|
"method": "iam",
|
|
99
150
|
"host": "redshift-cluster.region.redshift.amazonaws.com",
|
|
100
|
-
"
|
|
151
|
+
"database": "analytics",
|
|
101
152
|
"port": "5439",
|
|
102
153
|
"iamProfile": "redshift-access-role",
|
|
103
154
|
"region": "us-west-2",
|
|
@@ -29,13 +29,13 @@
|
|
|
29
29
|
}
|
|
30
30
|
],
|
|
31
31
|
"outputs": [
|
|
32
|
-
{ "format": "<api |
|
|
32
|
+
{ "format": "<api | sql>" },
|
|
33
33
|
{
|
|
34
|
-
"format": "<json | csv | parquet
|
|
34
|
+
"format": "<json | csv | parquet>",
|
|
35
35
|
"exportDestination": "<export destination>"
|
|
36
36
|
},
|
|
37
37
|
{
|
|
38
|
-
"format": "<csv>",
|
|
38
|
+
"format": "<csv | json>",
|
|
39
39
|
"exportDestination": "<export destination>",
|
|
40
40
|
"trigger": {
|
|
41
41
|
"type": "<cron | api>",
|
|
@@ -58,7 +58,6 @@ class RedshiftDriver {
|
|
|
58
58
|
const executeStatementRes = yield this._client.executeStatement(input);
|
|
59
59
|
const queryId = (_a = executeStatementRes.Id) !== null && _a !== void 0 ? _a : '';
|
|
60
60
|
const res = (yield this.executeStatement(queryId, { retry: 5 }))[0];
|
|
61
|
-
console.log(res);
|
|
62
61
|
return {
|
|
63
62
|
rows: res.result
|
|
64
63
|
};
|
|
@@ -77,7 +76,6 @@ class RedshiftDriver {
|
|
|
77
76
|
const queryId = (_b = executeStatementRes.Id) !== null && _b !== void 0 ? _b : '';
|
|
78
77
|
// TODO: handle errors (permission, wrong query, ...)
|
|
79
78
|
const res = (yield this.executeStatement(queryId, { retry: 5 }))[0];
|
|
80
|
-
console.log(res);
|
|
81
79
|
return {
|
|
82
80
|
rows: res.result
|
|
83
81
|
};
|
package/drivers/S3Driver.js
CHANGED
|
@@ -18,7 +18,7 @@ const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
|
|
|
18
18
|
class S3Driver {
|
|
19
19
|
constructor() {
|
|
20
20
|
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
|
|
21
|
-
this._bucketName = source.authentication['
|
|
21
|
+
this._bucketName = source.authentication['bucket'];
|
|
22
22
|
const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
|
|
23
23
|
const config = {
|
|
24
24
|
region: source.authentication['region'],
|
|
@@ -26,7 +26,7 @@ const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
|
|
|
26
26
|
class S3SourceDriver {
|
|
27
27
|
constructor() {
|
|
28
28
|
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
|
|
29
|
-
this._bucketName = source.authentication['
|
|
29
|
+
this._bucketName = source.authentication['bucket'];
|
|
30
30
|
const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
|
|
31
31
|
const config = {
|
|
32
32
|
region: source.authentication['region'],
|
|
@@ -105,7 +105,8 @@ class FileExporterClass {
|
|
|
105
105
|
}
|
|
106
106
|
case 'aws-s3': {
|
|
107
107
|
const driver = yield DriverFactory_1.default.instantiateDestination(source);
|
|
108
|
-
const
|
|
108
|
+
const date = DSTE_1.default.now().toISOString().split('.')[0];
|
|
109
|
+
const res = yield driver.uploadFile({ content: exportData, name: `${consumer.name}_${date}.${extension}` });
|
|
109
110
|
return res.key;
|
|
110
111
|
}
|
|
111
112
|
}
|