@hotglue/cli 1.0.31 → 1.0.33

package/lib/commands/etl/deploy.js

@@ -1,263 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.handler = exports.desc = exports.command = exports.builder = void 0;
-
-var _path = _interopRequireDefault(require("path"));
-
-var _promises = require("fs/promises");
-
-var _debug = _interopRequireDefault(require("../../helpers/debug"));
-
-var _ora = _interopRequireDefault(require("ora"));
-
-var _axios = _interopRequireDefault(require("axios/dist/node/axios.cjs"));
-
-var _awsSdk = _interopRequireDefault(require("aws-sdk"));
-
-var _cliTable = _interopRequireDefault(require("cli-table"));
-
-var _utils = require("../../helpers/utils");
-
-var _descriptions = _interopRequireDefault(require("../../helpers/descriptions"));
-
-var _print = require("../../helpers/print");
-
-var _api = require("../../helpers/api");
-
-var _flow = require("../../helpers/flow");
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-const debug = (0, _debug.default)('commands:etl:deploy');
-const command = 'deploy';
-exports.command = command;
-const desc = 'Deploy ETL scripts';
-exports.desc = desc;
-
-const builder = async yargs => {
-  debug('builder', command);
-  return yargs.option('flow', _descriptions.default.options['flow'].config).demandOption('flow', _descriptions.default.options['flow'].demandText).option('tap', _descriptions.default.options['tap'].config).option('all', _descriptions.default.options['all'].config).option('tenant', { ..._descriptions.default.options['tenant'].config,
-    default: 'default'
-  }).option('sourceFolder', _descriptions.default.options['sourceFolder'].config).demandOption('sourceFolder', _descriptions.default.options['sourceFolder'].demandText);
-};
-
-exports.builder = builder;
-
-const handler = async argv => {
-  debug('handler', command, argv);
-  const {
-    hg,
-    json,
-    apikey,
-    env,
-    flow,
-    tap,
-    all,
-    tenant,
-    sourceFolder
-  } = argv;
-  const {
-    clientApiBaseUri
-  } = hg;
-
-  if (!all && !tap) {
-    throw new Error(`${_descriptions.default.options['tap'].demandText} Or you can pass the --all flag to run this command for all taps/connectors.`);
-  }
-
-  let message;
-  let spinner = (0, _ora.default)();
-
-  const folderPrefix = _path.default.resolve(process.cwd(), sourceFolder);
-
-  try {
-    message = (0, _print.themed)(`Deploying script for Tenant ${(0, _print.themed)(tenant, 'info')} Flow ${(0, _print.themed)(flow, 'info')}${tap ? ` and Tap ${(0, _print.themed)(tap, 'info')}` : ''} to ${(0, _print.themed)(env, 'info')}`);
-    !json && spinner.info((0, _print.themed)(`Info: ${message}.`, 'secondary')); // 1. build list of deployable files
-
-    const deployableFiles = await (0, _utils.getFolderFiles)(folderPrefix, {
-      recursive: true,
-      filter: {
-        pattern: '!((**/sync-output)|(**/etl-output)|(**/snapshots))'
-      }
-    });
-
-    if (deployableFiles.length === 0) {
-      json ? (0, _print.printJSON)({
-        status: 'error',
-        error: 'There are no files to deploy at the specified location!'
-      }) : spinner.fail((0, _print.themed)(`Error: ${(0, _print.themed)('There are no files to deploy at the specified location!')}.`, 'secondary'));
-      return;
-    } // 2. get STS credentials
-
-
-    message = (0, _print.themed)(`Verifying user and authorizing`);
-    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
-    const params = {
-      debug,
-      baseUri: clientApiBaseUri,
-      task: 'etl-deploy',
-      env,
-      tenant,
-      flow,
-      apikey
-    };
-    if (tap) params.tap = tap;
-    const {
-      accessKeyId,
-      secretAccessKey,
-      sessionToken
-    } = await (0, _api.genCredentialsOnClientApi)(params);
-    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // create authenticated S3 instance
-
-    const s3 = new _awsSdk.default.S3({
-      accessKeyId,
-      secretAccessKey,
-      sessionToken
-    });
-    const isV2 = await (0, _flow.isV2Flow)(s3, env, tenant, flow); // 3. verify location is valid (consider ListObjects policy + listObjectsV2 + scan folder for key = config.json)
-
-    message = (0, _print.themed)(`Validating flow and tap location`);
-    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
-    const supportedFlows = await (0, _api.getSupportedFlows)({
-      debug,
-      baseUri: clientApiBaseUri,
-      apikey,
-      env
-    });
-    const supportedFlow = supportedFlows.find(({
-      id
-    }) => id === flow);
-    let connectors = [];
-
-    try {
-      if (tenant === 'default' || supportedFlow !== null && supportedFlow !== void 0 && supportedFlow.type) {
-        const supportedSources = isV2 ? await (0, _api.getSupportedConnectors)({
-          debug,
-          baseUri: clientApiBaseUri,
-          env,
-          flow,
-          apikey
-        }) : await (0, _api.getSupportedSources)({
-          debug,
-          baseUri: clientApiBaseUri,
-          env,
-          flow,
-          apikey
-        });
-        connectors = (supportedSources ?? []).filter(ss => tap ? ss[isV2 ? 'id' : 'tap'] === tap : true);
-        if (tap && connectors.length === 0) throw new Error('Tap is not supported');
-      } else {
-        const linkedSources = isV2 ? await (0, _api.getLinkedConnectors)({
-          debug,
-          baseUri: clientApiBaseUri,
-          env,
-          flow,
-          tenant,
-          apikey
-        }) : await (0, _api.getLinkedSources)({
-          debug,
-          baseUri: clientApiBaseUri,
-          env,
-          flow,
-          tenant,
-          apikey
-        });
-        connectors = (linkedSources ?? []).filter(ls => tap ? ls[isV2 ? 'id' : 'tap'] === tap : true);
-        if (tap && connectors.length === 0) throw new Error('Tap is not linked');
-      }
-    } catch (err) {
-      debug('err', err);
-      throw new Error(`Target location doesn't exist. Check your tenant, flow and tap arguments.`);
-    }
-
-    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
-    const table = new _cliTable.default({
-      head: ['File', 'Status']
-    });
-
-    for (const connector of connectors) {
-      const connectorId = connector[isV2 ? 'id' : 'tap'];
-      message = (0, _print.themed)(`Preparing ${connectorId} deployment target`);
-      !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
-      const rootKey = `${tenant}/flows/${flow}/${await (0, _flow.getEntityLabel)(null, null, null, null, isV2)}/${connectorId}/etl/`;
-      const {
-        Contents
-      } = await s3.listObjectsV2({
-        Bucket: env,
-        Prefix: `${rootKey}`
-      }).promise();
-      !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
-      debug('contents', Contents); // 4. cleanup old files
-
-      const filesToDelete = Contents.map(item => ({
-        Key: item.Key
-      }));
-
-      if (filesToDelete.length > 0) {
-        message = (0, _print.themed)(`Removing old ${connectorId} ETL files`);
-        !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
-        const delParams = {
-          Bucket: env,
-          Delete: {
-            Objects: filesToDelete,
-            Quiet: true
-          }
-        };
-        await s3.deleteObjects(delParams).promise();
-        filesToDelete.forEach(({
-          Key
-        }) => table.push([`${connectorId}/${Key.substring(rootKey.length)}`, (0, _print.themed)('Deleted', 'warn')]));
-        !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
-      } // 5. upload new files
-      // message = themed(`Deploying ETL scripts`);
-      // !json && spinner.info(themed(`Info: ${message}.`, 'secondary'));
-
-
-      for await (const file of deployableFiles) {
-        const relativePath = process.platform === 'win32' ? _path.default.relative(sourceFolder, file).replace(/\\/g, '/') : _path.default.relative(sourceFolder, file);
-        const key = `${rootKey}${relativePath}`;
-        message = (0, _print.themed)(`Deploying file: ${(0, _print.themed)(`${connectorId}/${relativePath}`, 'info')}`);
-        !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
-        const readBuffer = await (0, _promises.readFile)(file);
-        const params = {
-          Bucket: env,
-          Key: key,
-          Body: readBuffer
-        };
-        const res = await s3.putObject(params).promise();
-        !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
-        debug('s3-put-res', res);
-        table.push([`${connectorId}/${relativePath}`, (0, _print.themed)('Deployed', 'info')]);
-      }
-    }
-
-    if (json) {
-      (0, _print.printJSON)({
-        status: 'success',
-        deployedFiles: deployableFiles
-      });
-    } else {
-      // print results
-      (0, _print.cl)(table.toString());
-    }
-  } catch (err) {
-    if (json) {
-      (0, _print.printJSON)({
-        status: 'error',
-        error: err
-      });
-    } else {
-      spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
-      (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
-      debug(err);
-
-      if (err && err.response && err.response.data) {
-        debug('response', err.response.data);
-      }
-    }
-  }
-};
-
-exports.handler = handler;

package/lib/commands/etl/download.js

@@ -1,205 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.handler = exports.desc = exports.command = exports.builder = void 0;
-
-var _path = _interopRequireDefault(require("path"));
-
-var _promises = require("fs/promises");
-
-var _fs = require("fs");
-
-var _debug = _interopRequireDefault(require("../../helpers/debug"));
-
-var _ora = _interopRequireDefault(require("ora"));
-
-var _axios = _interopRequireDefault(require("axios/dist/node/axios.cjs"));
-
-var _cliTable = _interopRequireDefault(require("cli-table"));
-
-var _awsSdk = _interopRequireDefault(require("aws-sdk"));
-
-var _progress = _interopRequireDefault(require("progress"));
-
-var _descriptions = _interopRequireDefault(require("../../helpers/descriptions"));
-
-var _print = _interopRequireWildcard(require("../../helpers/print"));
-
-var _utils = require("../../helpers/utils");
-
-var _api = require("../../helpers/api");
-
-var _flow = require("../../helpers/flow");
-
-function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
-
-function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-const debug = (0, _debug.default)('commands:etl:download');
-const command = 'download';
-exports.command = command;
-const desc = 'Download ETL scripts';
-exports.desc = desc;
-
-const builder = async yargs => {
-  debug('builder', command);
-  return yargs.option('flow', _descriptions.default.options['flow'].config).demandOption('flow', _descriptions.default.options['flow'].demandText).option('tap', _descriptions.default.options['tap'].config).demandOption('tap', _descriptions.default.options['tap'].demandText).option('downloadTo', _descriptions.default.options['downloadTo'].config).demandOption('downloadTo', _descriptions.default.options['downloadTo'].demandText).option('tenant', { ..._descriptions.default.options['tenant'].config,
-    default: 'default'
-  }).option('overwrite', _descriptions.default.options['overwrite'].config).demandOption('overwrite', _descriptions.default.options['overwrite'].demandText);
-};
-
-exports.builder = builder;
-
-const handler = async argv => {
-  debug('handler', command, argv);
-  const {
-    hg,
-    json,
-    apikey,
-    env,
-    flow,
-    tap,
-    tenant,
-    downloadTo,
-    overwrite
-  } = argv;
-  const {
-    clientApiBaseUri
-  } = hg;
-  let message;
-  let spinner = (0, _ora.default)();
-
-  const folderPrefix = _path.default.resolve(process.cwd(), downloadTo);
-
-  try {
-    // 1. get STS credentials
-    message = (0, _print.themed)(`Verifying user and authorizing`);
-    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
-    const {
-      accessKeyId,
-      secretAccessKey,
-      sessionToken
-    } = await (0, _api.genCredentialsOnClientApi)({
-      debug,
-      baseUri: clientApiBaseUri,
-      task: 'etl-download',
-      env,
-      tenant,
-      flow,
-      tap,
-      apikey
-    }); // created authenticated S3 instance
-
-    const s3 = new _awsSdk.default.S3({
-      accessKeyId,
-      secretAccessKey,
-      sessionToken
-    });
-    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // 2. get list of downloadable files
-
-    message = (0, _print.themed)(`Scanning for downloadable files`);
-    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
-    const keyPrefix = `${tenant}/flows/${flow}/${await (0, _flow.getEntityLabel)(s3, env, tenant, flow)}/${tap}/etl/`;
-    const params = {
-      Bucket: env,
-      Prefix: keyPrefix
-    };
-    const {
-      Contents: s3Files
-    } = await s3.listObjectsV2(params).promise();
-    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
-    debug('s3-list-res', s3Files); // filter unwanted files from source
-
-    const filesToDownload = s3Files ? (0, _utils.filterFiles)(s3Files.map(item => item.Key), {
-      pattern: '!(.ipynb_checkpoints/*)'
-    }) : [];
-
-    if (!filesToDownload || filesToDownload.length === 0) {
-      json ? (0, _print.printJSON)({
-        status: 'success',
-        downloadedFiles: []
-      }) : spinner.warn((0, _print.themed)(`Warning: ${(0, _print.themed)('Nothing to download!')}`, 'secondary'));
-      return;
-    } // 3. download files
-
-
-    message = (0, _print.themed)(`Downloading script files to ${(0, _print.themed)(folderPrefix, 'info')}`);
-    !json && spinner.info((0, _print.themed)(`Info: ${message}.`, 'secondary'));
-    const table = new _cliTable.default({
-      head: ['File', 'Status']
-    });
-
-    for await (const file of filesToDownload) {
-      const subPath = file.substring(keyPrefix.length); // path.basename(`${env}/${item.Key}`);
-
-      const fullPath = _path.default.resolve(folderPrefix, subPath);
-
-      debug('file', fullPath);
-
-      if (file.endsWith('/')) {
-        !(0, _fs.existsSync)(fullPath) && (await (0, _promises.mkdir)(fullPath, {
-          recursive: true
-        }));
-        continue;
-      } // skip if exists and overwrite is false
-
-
-      if (!overwrite) {
-        try {
-          await (0, _promises.stat)(fullPath);
-          debug('exists, skipping');
-          table.push([subPath, 'Skipped']);
-          continue;
-        } catch (err) {
-          if (err.code !== 'ENOENT') (0, _print.cl)(err);
-        }
-      }
-
-      message = (0, _print.themed)(`Downloading file: ${(0, _print.themed)(subPath, 'info')}`);
-      !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary')); // ensure destination folder exists
-
-      await (0, _promises.mkdir)(_path.default.dirname(fullPath), {
-        recursive: true
-      }); // download s3 object into destination folder
-
-      const fileStream = s3.getObject({
-        Bucket: env,
-        Key: file
-      }).createReadStream();
-      await (0, _promises.writeFile)(fullPath, await (0, _utils.streamToString)(fileStream));
-      !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
-      table.push([subPath, 'Downloaded']);
-    }
-
-    if (json) {
-      (0, _print.printJSON)({
-        status: 'success',
-        downloadedFiles: filesToDownload
-      });
-    } else {
-      // print results
-      (0, _print.cl)(table.toString());
-    }
-  } catch (err) {
-    if (json) {
-      (0, _print.printJSON)({
-        status: 'error',
-        error: err
-      });
-    } else {
-      spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
-      (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
-      debug(err);
-
-      if (err && err.response && err.response.data) {
-        debug('response', err.response.data);
-      }
-    }
-  }
-};
-
-exports.handler = handler;

package/lib/commands/etl.js

@@ -1,32 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.handler = exports.desc = exports.command = exports.builder = void 0;
-
-var _debug = _interopRequireDefault(require("../helpers/debug"));
-
-var _baseBuilder = require("../helpers/baseBuilder");
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-const debug = (0, _debug.default)('commands:etl');
-const command = 'etl <action>';
-exports.command = command;
-const desc = 'Manage ETL scripts';
-exports.desc = desc;
-
-const builder = async function (yargs) {
-  debug('builder', command);
-  const base = await (0, _baseBuilder.baseApiBuilder)(yargs);
-  return base.commandDir('etl');
-};
-
-exports.builder = builder;
-
-const handler = async function (argv) {
-  debug('handler', command, argv);
-};
-
-exports.handler = handler;

package/lib/commands/flows/list.js

@@ -1,104 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.handler = exports.desc = exports.command = exports.builder = void 0;
-
-var _util = require("util");
-
-var _debug = _interopRequireDefault(require("../../helpers/debug"));
-
-var _ora = _interopRequireDefault(require("ora"));
-
-var _axios = _interopRequireDefault(require("axios/dist/node/axios.cjs"));
-
-var _cliTable = _interopRequireDefault(require("cli-table"));
-
-var _descriptions = _interopRequireDefault(require("../../helpers/descriptions"));
-
-var _print = require("../../helpers/print");
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-const debug = (0, _debug.default)('commands:flows:list');
-const command = 'list';
-exports.command = command;
-const desc = 'List flows';
-exports.desc = desc;
-
-const builder = async yargs => {
-  debug('builder', command);
-  return yargs.option('tenant', { ..._descriptions.default.options['tenant'].config
-  });
-};
-
-exports.builder = builder;
-
-const handler = async argv => {
-  debug('handler', command, argv);
-  const {
-    hg,
-    json,
-    apikey,
-    env,
-    tenant
-  } = argv;
-  const {
-    clientApiBaseUri
-  } = hg;
-  let message;
-  let spinner = (0, _ora.default)();
-
-  try {
-    // 1. get data from client-api backend
-    message = (0, _print.themed)(`Retrieving ${!tenant ? `the` : `${(0, _print.themed)(tenant, 'info')} tenant's`} flows for environment: ${(0, _print.themed)(env, 'info')}`); // todo: look into changing when if client-api combined these two endpoints
-
-    const uri = `${clientApiBaseUri}/${env}/flows/${tenant ? `linked?user_id=${tenant}` : `supported`}`;
-    debug('requesting:', uri);
-    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
-    const {
-      data
-    } = await _axios.default.get(uri, {
-      headers: {
-        'x-api-key': apikey
-      }
-    });
-    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
-    debug('response-data', data); // exit if there is no data
-
-    if (!data || data.length === 0) {
-      json ? (0, _print.printJSON)([]) : spinner.warn((0, _print.themed)(`Warning: ${(0, _print.themed)('No flows for specified environment and tenant')}.`, 'secondary'));
-      return;
-    }
-
-    if (json) {
-      (0, _print.printJSON)(data);
-    } else {
-      // generate results table
-      const table = new _cliTable.default({
-        head: ['ID', 'Name', 'isPush', 'Taps', 'Targets']
-      });
-      data.forEach(item => table.push([item.id, item.name || '', item.type || false, item.taps ? (0, _util.inspect)(item.taps) : '', item.targets ? (0, _util.inspect)(item.targets) : 'none'])); // print results
-
-      console.log(table.toString());
-    }
-  } catch (err) {
-    if (json) {
-      (0, _print.printJSON)({
-        status: 'error',
-        error: err
-      });
-    } else {
-      spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
-      (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
-      debug(err);
-
-      if (err && err.response && err.response.data) {
-        debug('response', err.response.data);
-      }
-    }
-  }
-};
-
-exports.handler = handler;

package/lib/commands/flows.js

@@ -1,32 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.handler = exports.desc = exports.command = exports.builder = void 0;
-
-var _debug = _interopRequireDefault(require("../helpers/debug"));
-
-var _baseBuilder = require("../helpers/baseBuilder");
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-const debug = (0, _debug.default)('commands:flows');
-const command = 'flows <action>';
-exports.command = command;
-const desc = 'Manage flows';
-exports.desc = desc;
-
-const builder = async function (yargs) {
-  debug('builder', command);
-  const base = await (0, _baseBuilder.baseApiBuilder)(yargs);
-  return base.commandDir('flows');
-};
-
-exports.builder = builder;
-
-const handler = async function (argv) {
-  debug('handler', command, argv);
-};
-
-exports.handler = handler;