@hotglue/cli 1.0.32 → 1.0.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +1 -1
  2. package/lib/cjs/index.js +9 -8
  3. package/lib/cjs/index.js.map +1 -1
  4. package/lib/mjs/index.js +9 -8
  5. package/lib/mjs/index.js.map +1 -1
  6. package/package.json +2 -1
  7. package/lib/commands/config/set.js +0 -82
  8. package/lib/commands/config.js +0 -69
  9. package/lib/commands/env/deploy.js +0 -162
  10. package/lib/commands/env/download.js +0 -169
  11. package/lib/commands/env.js +0 -32
  12. package/lib/commands/etl/delete.js +0 -171
  13. package/lib/commands/etl/deploy.js +0 -263
  14. package/lib/commands/etl/download.js +0 -205
  15. package/lib/commands/etl.js +0 -32
  16. package/lib/commands/flows/list.js +0 -104
  17. package/lib/commands/flows.js +0 -32
  18. package/lib/commands/jobs/download.js +0 -188
  19. package/lib/commands/jobs/list.js +0 -108
  20. package/lib/commands/jobs.js +0 -32
  21. package/lib/commands/snapshots/deploy.js +0 -202
  22. package/lib/commands/snapshots/download.js +0 -213
  23. package/lib/commands/snapshots.js +0 -32
  24. package/lib/commands/tenants/custom-etl.js +0 -183
  25. package/lib/commands/tenants/custom-field-map.js +0 -191
  26. package/lib/commands/tenants/delete.js +0 -150
  27. package/lib/commands/tenants/list.js +0 -97
  28. package/lib/commands/tenants/updateConfig.js +0 -188
  29. package/lib/commands/tenants.js +0 -32
  30. package/lib/helpers/api.js +0 -326
  31. package/lib/helpers/baseBuilder.js +0 -30
  32. package/lib/helpers/config.js +0 -105
  33. package/lib/helpers/debug.js +0 -19
  34. package/lib/helpers/descriptions.js +0 -137
  35. package/lib/helpers/flow.js +0 -32
  36. package/lib/helpers/print.js +0 -92
  37. package/lib/helpers/utils.js +0 -245
  38. package/lib/index.js +0 -12
  39. package/lib/main.js +0 -27
@@ -1,171 +0,0 @@
1
- "use strict";
2
-
3
- Object.defineProperty(exports, "__esModule", {
4
- value: true
5
- });
6
- exports.handler = exports.desc = exports.command = exports.builder = void 0;
7
-
8
- var _debug = _interopRequireDefault(require("../../helpers/debug"));
9
-
10
- var _ora = _interopRequireDefault(require("ora"));
11
-
12
- var _awsSdk = _interopRequireDefault(require("aws-sdk"));
13
-
14
- var _cliTable = _interopRequireDefault(require("cli-table"));
15
-
16
- var _descriptions = _interopRequireDefault(require("../../helpers/descriptions"));
17
-
18
- var _print = require("../../helpers/print");
19
-
20
- var _api = require("../../helpers/api");
21
-
22
- var _flow = require("../../helpers/flow");
23
-
24
- function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
25
-
26
- const debug = (0, _debug.default)('commands:etl:delete');
27
- const command = 'delete';
28
- exports.command = command;
29
- const desc = 'Delete ETL scripts';
30
- exports.desc = desc;
31
-
32
- const builder = async yargs => {
33
- debug('builder', command);
34
- return yargs.option('flow', _descriptions.default.options['flow'].config).demandOption('flow', _descriptions.default.options['flow'].demandText).option('tap', _descriptions.default.options['tap'].config).demandOption('tap', _descriptions.default.options['tap'].demandText).option('tenant', { ..._descriptions.default.options['tenant'].config
35
- }).demandOption('tenant', _descriptions.default.options['tenant'].demandText);
36
- };
37
-
38
- exports.builder = builder;
39
-
40
- const handler = async argv => {
41
- debug('handler', command, argv);
42
- const {
43
- hg,
44
- json,
45
- apikey,
46
- env,
47
- flow,
48
- tap,
49
- tenant
50
- } = argv;
51
- const {
52
- clientApiBaseUri
53
- } = hg;
54
- let message;
55
- let spinner = (0, _ora.default)();
56
-
57
- try {
58
- var _listObjectsRes$Conte, _listObjectsRes$Conte2;
59
-
60
- message = (0, _print.themed)(`Deleting ETL scripts for Tenant ${(0, _print.themed)(tenant, 'info')} Flow ${(0, _print.themed)(flow, 'info')} and Tap ${(0, _print.themed)(tap, 'info')} to ${(0, _print.themed)(env, 'info')}`);
61
- !json && spinner.info((0, _print.themed)(`Info: ${message}.`, 'secondary')); // 1. get STS credentials
62
-
63
- message = (0, _print.themed)(`Verifying user and authorizing`);
64
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
65
- const {
66
- accessKeyId,
67
- secretAccessKey,
68
- sessionToken
69
- } = await (0, _api.genCredentialsOnClientApi)({
70
- debug,
71
- baseUri: clientApiBaseUri,
72
- task: 'etl-deploy',
73
- env,
74
- tenant,
75
- flow,
76
- tap,
77
- apikey
78
- }); // created authenticated S3 instance
79
-
80
- const s3 = new _awsSdk.default.S3({
81
- accessKeyId,
82
- secretAccessKey,
83
- sessionToken
84
- });
85
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // 2. delete ETL folder
86
-
87
- message = (0, _print.themed)(`Deleting ETL scripts`);
88
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
89
- const keyPrefix = `${tenant}/flows/${flow}/${await (0, _flow.getEntityLabel)(s3, env, tenant, flow)}/${tap}/etl/`; // 2.1 Get all objects from folder
90
-
91
- const listObjectsRes = await s3.listObjectsV2({
92
- Bucket: env,
93
- Prefix: keyPrefix
94
- }).promise();
95
- debug('s3-list-objects-res', listObjectsRes);
96
-
97
- if (((_listObjectsRes$Conte = listObjectsRes.Contents) === null || _listObjectsRes$Conte === void 0 ? void 0 : _listObjectsRes$Conte.length) > 0) {
98
- // 2.2 Delete all objects
99
- const deleteObjectsRes = await s3.deleteObjects({
100
- Bucket: env,
101
- Delete: {
102
- Objects: listObjectsRes.Contents.map(object => {
103
- return {
104
- Key: object.Key
105
- };
106
- })
107
- }
108
- }).promise();
109
- debug('s3-list-objects-res', deleteObjectsRes);
110
- } // 2.3 Delete the folder if needed
111
-
112
-
113
- const response = await s3.deleteObject({
114
- Bucket: env,
115
- Key: keyPrefix
116
- }).promise();
117
- debug('s3-delete-object-res', response);
118
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
119
-
120
- if (((_listObjectsRes$Conte2 = listObjectsRes.Contents) === null || _listObjectsRes$Conte2 === void 0 ? void 0 : _listObjectsRes$Conte2.length) > 0) {
121
- const table = new _cliTable.default({
122
- head: ['File', 'Status']
123
- });
124
- listObjectsRes.Contents.filter(({
125
- Key
126
- }) => {
127
- return Key !== keyPrefix;
128
- }).forEach(({
129
- Key
130
- }) => {
131
- const splittedKey = Key.split('/');
132
- const filename = splittedKey[splittedKey.length - 1];
133
- table.push([filename, (0, _print.themed)('Deleted', 'info')]);
134
- });
135
-
136
- if (json) {
137
- (0, _print.printJSON)({
138
- status: 'success',
139
- deletedObjects: (listObjectsRes.Contents ?? []).map(({
140
- Key
141
- }) => Key)
142
- });
143
- } else {
144
- (0, _print.cl)(table.toString());
145
- }
146
- } else {
147
- message = (0, _print.themed)(`There was no file or folder to be deleted`);
148
- json ? (0, _print.printJSON)({
149
- status: 'success',
150
- deletedObjects: []
151
- }) : spinner.info((0, _print.themed)(`Info: ${message}.`, 'secondary'));
152
- }
153
- } catch (err) {
154
- if (json) {
155
- (0, _print.printJSON)({
156
- status: 'error',
157
- error: err
158
- });
159
- } else {
160
- spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
161
- (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
162
- debug(err);
163
-
164
- if (err && err.response && err.response.data) {
165
- debug('response', err.response.data);
166
- }
167
- }
168
- }
169
- };
170
-
171
- exports.handler = handler;
@@ -1,263 +0,0 @@
1
- "use strict";
2
-
3
- Object.defineProperty(exports, "__esModule", {
4
- value: true
5
- });
6
- exports.handler = exports.desc = exports.command = exports.builder = void 0;
7
-
8
- var _path = _interopRequireDefault(require("path"));
9
-
10
- var _promises = require("fs/promises");
11
-
12
- var _debug = _interopRequireDefault(require("../../helpers/debug"));
13
-
14
- var _ora = _interopRequireDefault(require("ora"));
15
-
16
- var _axios = _interopRequireDefault(require("axios/dist/node/axios.cjs"));
17
-
18
- var _awsSdk = _interopRequireDefault(require("aws-sdk"));
19
-
20
- var _cliTable = _interopRequireDefault(require("cli-table"));
21
-
22
- var _utils = require("../../helpers/utils");
23
-
24
- var _descriptions = _interopRequireDefault(require("../../helpers/descriptions"));
25
-
26
- var _print = require("../../helpers/print");
27
-
28
- var _api = require("../../helpers/api");
29
-
30
- var _flow = require("../../helpers/flow");
31
-
32
- function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
33
-
34
- const debug = (0, _debug.default)('commands:etl:deploy');
35
- const command = 'deploy';
36
- exports.command = command;
37
- const desc = 'Deploy ETL scripts';
38
- exports.desc = desc;
39
-
40
- const builder = async yargs => {
41
- debug('builder', command);
42
- return yargs.option('flow', _descriptions.default.options['flow'].config).demandOption('flow', _descriptions.default.options['flow'].demandText).option('tap', _descriptions.default.options['tap'].config).option('all', _descriptions.default.options['all'].config).option('tenant', { ..._descriptions.default.options['tenant'].config,
43
- default: 'default'
44
- }).option('sourceFolder', _descriptions.default.options['sourceFolder'].config).demandOption('sourceFolder', _descriptions.default.options['sourceFolder'].demandText);
45
- };
46
-
47
- exports.builder = builder;
48
-
49
- const handler = async argv => {
50
- debug('handler', command, argv);
51
- const {
52
- hg,
53
- json,
54
- apikey,
55
- env,
56
- flow,
57
- tap,
58
- all,
59
- tenant,
60
- sourceFolder
61
- } = argv;
62
- const {
63
- clientApiBaseUri
64
- } = hg;
65
-
66
- if (!all && !tap) {
67
- throw new Error(`${_descriptions.default.options['tap'].demandText} Or you can pass the --all flag to run this command for all taps/connectors.`);
68
- }
69
-
70
- let message;
71
- let spinner = (0, _ora.default)();
72
-
73
- const folderPrefix = _path.default.resolve(process.cwd(), sourceFolder);
74
-
75
- try {
76
- message = (0, _print.themed)(`Deploying script for Tenant ${(0, _print.themed)(tenant, 'info')} Flow ${(0, _print.themed)(flow, 'info')}${tap ? ` and Tap ${(0, _print.themed)(tap, 'info')}` : ''} to ${(0, _print.themed)(env, 'info')}`);
77
- !json && spinner.info((0, _print.themed)(`Info: ${message}.`, 'secondary')); // 1. build list of deployable files
78
-
79
- const deployableFiles = await (0, _utils.getFolderFiles)(folderPrefix, {
80
- recursive: true,
81
- filter: {
82
- pattern: '!((**/sync-output)|(**/etl-output)|(**/snapshots))'
83
- }
84
- });
85
-
86
- if (deployableFiles.length === 0) {
87
- json ? (0, _print.printJSON)({
88
- status: 'error',
89
- error: 'There are no files to deploy at the specified location!'
90
- }) : spinner.fail((0, _print.themed)(`Error: ${(0, _print.themed)('There are no files to deploy at the specified location!')}.`, 'secondary'));
91
- return;
92
- } // 2. get STS credentials
93
-
94
-
95
- message = (0, _print.themed)(`Verifying user and authorizing`);
96
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
97
- const params = {
98
- debug,
99
- baseUri: clientApiBaseUri,
100
- task: 'etl-deploy',
101
- env,
102
- tenant,
103
- flow,
104
- apikey
105
- };
106
- if (tap) params.tap = tap;
107
- const {
108
- accessKeyId,
109
- secretAccessKey,
110
- sessionToken
111
- } = await (0, _api.genCredentialsOnClientApi)(params);
112
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // create authenticated S3 instance
113
-
114
- const s3 = new _awsSdk.default.S3({
115
- accessKeyId,
116
- secretAccessKey,
117
- sessionToken
118
- });
119
- const isV2 = await (0, _flow.isV2Flow)(s3, env, tenant, flow); // 3. verify location is valid (consider ListObjects policy + listObjectsV2 + scan folder for key = config.json)
120
-
121
- message = (0, _print.themed)(`Validating flow and tap location`);
122
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
123
- const supportedFlows = await (0, _api.getSupportedFlows)({
124
- debug,
125
- baseUri: clientApiBaseUri,
126
- apikey,
127
- env
128
- });
129
- const supportedFlow = supportedFlows.find(({
130
- id
131
- }) => id === flow);
132
- let connectors = [];
133
-
134
- try {
135
- if (tenant === 'default' || supportedFlow !== null && supportedFlow !== void 0 && supportedFlow.type) {
136
- const supportedSources = isV2 ? await (0, _api.getSupportedConnectors)({
137
- debug,
138
- baseUri: clientApiBaseUri,
139
- env,
140
- flow,
141
- apikey
142
- }) : await (0, _api.getSupportedSources)({
143
- debug,
144
- baseUri: clientApiBaseUri,
145
- env,
146
- flow,
147
- apikey
148
- });
149
- connectors = (supportedSources ?? []).filter(ss => tap ? ss[isV2 ? 'id' : 'tap'] === tap : true);
150
- if (tap && connectors.length === 0) throw new Error('Tap is not supported');
151
- } else {
152
- const linkedSources = isV2 ? await (0, _api.getLinkedConnectors)({
153
- debug,
154
- baseUri: clientApiBaseUri,
155
- env,
156
- flow,
157
- tenant,
158
- apikey
159
- }) : await (0, _api.getLinkedSources)({
160
- debug,
161
- baseUri: clientApiBaseUri,
162
- env,
163
- flow,
164
- tenant,
165
- apikey
166
- });
167
- connectors = (linkedSources ?? []).filter(ls => tap ? ls[isV2 ? 'id' : 'tap'] === tap : true);
168
- if (tap && connectors.length === 0) throw new Error('Tap is not linked');
169
- }
170
- } catch (err) {
171
- debug('err', err);
172
- throw new Error(`Target location doesn't exist. Check your tenant, flow and tap arguments.`);
173
- }
174
-
175
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
176
- const table = new _cliTable.default({
177
- head: ['File', 'Status']
178
- });
179
-
180
- for (const connector of connectors) {
181
- const connectorId = connector[isV2 ? 'id' : 'tap'];
182
- message = (0, _print.themed)(`Preparing ${connectorId} deployment target`);
183
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
184
- const rootKey = `${tenant}/flows/${flow}/${await (0, _flow.getEntityLabel)(null, null, null, null, isV2)}/${connectorId}/etl/`;
185
- const {
186
- Contents
187
- } = await s3.listObjectsV2({
188
- Bucket: env,
189
- Prefix: `${rootKey}`
190
- }).promise();
191
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
192
- debug('contents', Contents); // 4. cleanup old files
193
-
194
- const filesToDelete = Contents.map(item => ({
195
- Key: item.Key
196
- }));
197
-
198
- if (filesToDelete.length > 0) {
199
- message = (0, _print.themed)(`Removing old ${connectorId} ETL files`);
200
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
201
- const delParams = {
202
- Bucket: env,
203
- Delete: {
204
- Objects: filesToDelete,
205
- Quiet: true
206
- }
207
- };
208
- await s3.deleteObjects(delParams).promise();
209
- filesToDelete.forEach(({
210
- Key
211
- }) => table.push([`${connectorId}/${Key.substring(rootKey.length)}`, (0, _print.themed)('Deleted', 'warn')]));
212
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
213
- } // 5. upload new files
214
- // message = themed(`Deploying ETL scripts`);
215
- // !json && spinner.info(themed(`Info: ${message}.`, 'secondary'));
216
-
217
-
218
- for await (const file of deployableFiles) {
219
- const relativePath = process.platform === 'win32' ? _path.default.relative(sourceFolder, file).replace(/\\/g, '/') : _path.default.relative(sourceFolder, file);
220
- const key = `${rootKey}${relativePath}`;
221
- message = (0, _print.themed)(`Deploying file: ${(0, _print.themed)(`${connectorId}/${relativePath}`, 'info')}`);
222
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
223
- const readBuffer = await (0, _promises.readFile)(file);
224
- const params = {
225
- Bucket: env,
226
- Key: key,
227
- Body: readBuffer
228
- };
229
- const res = await s3.putObject(params).promise();
230
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
231
- debug('s3-put-res', res);
232
- table.push([`${connectorId}/${relativePath}`, (0, _print.themed)('Deployed', 'info')]);
233
- }
234
- }
235
-
236
- if (json) {
237
- (0, _print.printJSON)({
238
- status: 'success',
239
- deployedFiles: deployableFiles
240
- });
241
- } else {
242
- // print results
243
- (0, _print.cl)(table.toString());
244
- }
245
- } catch (err) {
246
- if (json) {
247
- (0, _print.printJSON)({
248
- status: 'error',
249
- error: err
250
- });
251
- } else {
252
- spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
253
- (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
254
- debug(err);
255
-
256
- if (err && err.response && err.response.data) {
257
- debug('response', err.response.data);
258
- }
259
- }
260
- }
261
- };
262
-
263
- exports.handler = handler;
@@ -1,205 +0,0 @@
1
- "use strict";
2
-
3
- Object.defineProperty(exports, "__esModule", {
4
- value: true
5
- });
6
- exports.handler = exports.desc = exports.command = exports.builder = void 0;
7
-
8
- var _path = _interopRequireDefault(require("path"));
9
-
10
- var _promises = require("fs/promises");
11
-
12
- var _fs = require("fs");
13
-
14
- var _debug = _interopRequireDefault(require("../../helpers/debug"));
15
-
16
- var _ora = _interopRequireDefault(require("ora"));
17
-
18
- var _axios = _interopRequireDefault(require("axios/dist/node/axios.cjs"));
19
-
20
- var _cliTable = _interopRequireDefault(require("cli-table"));
21
-
22
- var _awsSdk = _interopRequireDefault(require("aws-sdk"));
23
-
24
- var _progress = _interopRequireDefault(require("progress"));
25
-
26
- var _descriptions = _interopRequireDefault(require("../../helpers/descriptions"));
27
-
28
- var _print = _interopRequireWildcard(require("../../helpers/print"));
29
-
30
- var _utils = require("../../helpers/utils");
31
-
32
- var _api = require("../../helpers/api");
33
-
34
- var _flow = require("../../helpers/flow");
35
-
36
- function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
37
-
38
- function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
39
-
40
- function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
41
-
42
- const debug = (0, _debug.default)('commands:etl:download');
43
- const command = 'download';
44
- exports.command = command;
45
- const desc = 'Download ETL scripts';
46
- exports.desc = desc;
47
-
48
- const builder = async yargs => {
49
- debug('builder', command);
50
- return yargs.option('flow', _descriptions.default.options['flow'].config).demandOption('flow', _descriptions.default.options['flow'].demandText).option('tap', _descriptions.default.options['tap'].config).demandOption('tap', _descriptions.default.options['tap'].demandText).option('downloadTo', _descriptions.default.options['downloadTo'].config).demandOption('downloadTo', _descriptions.default.options['downloadTo'].demandText).option('tenant', { ..._descriptions.default.options['tenant'].config,
51
- default: 'default'
52
- }).option('overwrite', _descriptions.default.options['overwrite'].config).demandOption('overwrite', _descriptions.default.options['overwrite'].demandText);
53
- };
54
-
55
- exports.builder = builder;
56
-
57
- const handler = async argv => {
58
- debug('handler', command, argv);
59
- const {
60
- hg,
61
- json,
62
- apikey,
63
- env,
64
- flow,
65
- tap,
66
- tenant,
67
- downloadTo,
68
- overwrite
69
- } = argv;
70
- const {
71
- clientApiBaseUri
72
- } = hg;
73
- let message;
74
- let spinner = (0, _ora.default)();
75
-
76
- const folderPrefix = _path.default.resolve(process.cwd(), downloadTo);
77
-
78
- try {
79
- // 1. get STS credentials
80
- message = (0, _print.themed)(`Verifying user and authorizing`);
81
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
82
- const {
83
- accessKeyId,
84
- secretAccessKey,
85
- sessionToken
86
- } = await (0, _api.genCredentialsOnClientApi)({
87
- debug,
88
- baseUri: clientApiBaseUri,
89
- task: 'etl-download',
90
- env,
91
- tenant,
92
- flow,
93
- tap,
94
- apikey
95
- }); // created authenticated S3 instance
96
-
97
- const s3 = new _awsSdk.default.S3({
98
- accessKeyId,
99
- secretAccessKey,
100
- sessionToken
101
- });
102
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // 2. get list of downloadable files
103
-
104
- message = (0, _print.themed)(`Scanning for downloadable files`);
105
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
106
- const keyPrefix = `${tenant}/flows/${flow}/${await (0, _flow.getEntityLabel)(s3, env, tenant, flow)}/${tap}/etl/`;
107
- const params = {
108
- Bucket: env,
109
- Prefix: keyPrefix
110
- };
111
- const {
112
- Contents: s3Files
113
- } = await s3.listObjectsV2(params).promise();
114
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
115
- debug('s3-list-res', s3Files); // filter unwanted files from source
116
-
117
- const filesToDownload = s3Files ? (0, _utils.filterFiles)(s3Files.map(item => item.Key), {
118
- pattern: '!(.ipynb_checkpoints/*)'
119
- }) : [];
120
-
121
- if (!filesToDownload || filesToDownload.length === 0) {
122
- json ? (0, _print.printJSON)({
123
- status: 'success',
124
- downloadedFiles: []
125
- }) : spinner.warn((0, _print.themed)(`Warning: ${(0, _print.themed)('Nothing to download!')}`, 'secondary'));
126
- return;
127
- } // 3. download files
128
-
129
-
130
- message = (0, _print.themed)(`Downloading script files to ${(0, _print.themed)(folderPrefix, 'info')}`);
131
- !json && spinner.info((0, _print.themed)(`Info: ${message}.`, 'secondary'));
132
- const table = new _cliTable.default({
133
- head: ['File', 'Status']
134
- });
135
-
136
- for await (const file of filesToDownload) {
137
- const subPath = file.substring(keyPrefix.length); // path.basename(`${env}/${item.Key}`);
138
-
139
- const fullPath = _path.default.resolve(folderPrefix, subPath);
140
-
141
- debug('file', fullPath);
142
-
143
- if (file.endsWith('/')) {
144
- !(0, _fs.existsSync)(fullPath) && (await (0, _promises.mkdir)(fullPath, {
145
- recursive: true
146
- }));
147
- continue;
148
- } // skip if exists and overwrite is false
149
-
150
-
151
- if (!overwrite) {
152
- try {
153
- await (0, _promises.stat)(fullPath);
154
- debug('exists, skipping');
155
- table.push([subPath, 'Skipped']);
156
- continue;
157
- } catch (err) {
158
- if (err.code !== 'ENOENT') (0, _print.cl)(err);
159
- }
160
- }
161
-
162
- message = (0, _print.themed)(`Downloading file: ${(0, _print.themed)(subPath, 'info')}`);
163
- !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary')); // ensure destination folder exists
164
-
165
- await (0, _promises.mkdir)(_path.default.dirname(fullPath), {
166
- recursive: true
167
- }); // download s3 object into destination folder
168
-
169
- const fileStream = s3.getObject({
170
- Bucket: env,
171
- Key: file
172
- }).createReadStream();
173
- await (0, _promises.writeFile)(fullPath, await (0, _utils.streamToString)(fileStream));
174
- !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
175
- table.push([subPath, 'Downloaded']);
176
- }
177
-
178
- if (json) {
179
- (0, _print.printJSON)({
180
- status: 'success',
181
- downloadedFiles: filesToDownload
182
- });
183
- } else {
184
- // print results
185
- (0, _print.cl)(table.toString());
186
- }
187
- } catch (err) {
188
- if (json) {
189
- (0, _print.printJSON)({
190
- status: 'error',
191
- error: err
192
- });
193
- } else {
194
- spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
195
- (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
196
- debug(err);
197
-
198
- if (err && err.response && err.response.data) {
199
- debug('response', err.response.data);
200
- }
201
- }
202
- }
203
- };
204
-
205
- exports.handler = handler;
@@ -1,32 +0,0 @@
1
- "use strict";
2
-
3
- Object.defineProperty(exports, "__esModule", {
4
- value: true
5
- });
6
- exports.handler = exports.desc = exports.command = exports.builder = void 0;
7
-
8
- var _debug = _interopRequireDefault(require("../helpers/debug"));
9
-
10
- var _baseBuilder = require("../helpers/baseBuilder");
11
-
12
- function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
13
-
14
- const debug = (0, _debug.default)('commands:etl');
15
- const command = 'etl <action>';
16
- exports.command = command;
17
- const desc = 'Manage ETL scripts';
18
- exports.desc = desc;
19
-
20
- const builder = async function (yargs) {
21
- debug('builder', command);
22
- const base = await (0, _baseBuilder.baseApiBuilder)(yargs);
23
- return base.commandDir('etl');
24
- };
25
-
26
- exports.builder = builder;
27
-
28
- const handler = async function (argv) {
29
- debug('handler', command, argv);
30
- };
31
-
32
- exports.handler = handler;