@hotglue/cli 1.0.25

package/README.md

@@ -0,0 +1,94 @@
+# @hotglue/cli [![npm version](https://badge.fury.io/js/hotglue-cli.svg)](https://npmjs.com/package/@hotglue/cli)
+
+The [hotglue](https://hotglue.xyz) Command Line Interface (CLI)
+
+## Table of Contents
+
+- [Installation](#installation)
+- [Usage](#usage)
+- [Configuration](#configuration)
+- [Documentation](#documentation)
+
+## Installation
+
+You can install the CLI globally (recommended) using
+
+```zsh
+npm install -g @hotglue/cli
+```
+
+And then you can run it from anywhere using
+
+```zsh
+hotglue --help
+```
+
+Alternatively, you may install it locally in a project folder using
+
+```zsh
+npm install --save-dev @hotglue/cli
+```
+
+And then you can run the local installation from within that folder using
+
+```zsh
+npx hotglue --help
+```
+
+## Usage
+
+```zsh
+Usage: hotglue <command>
+
+Commands:
+  hotglue config                  View global CLI settings (ie. apikey)
+  hotglue config set              Configure global CLI settings (ie. apikey)
+  hotglue env download            Download environment settings (ie. requirements.txt)
+  hotglue env deploy              Deploy environment settings (ie. requirements.txt)
+  hotglue etl download            Download ETL script files
+  hotglue etl deploy              Deploy ETL script files
+  hotglue snapshots download      Download snapshot files
+  hotglue snapshots deploy        Deploy snapshot files
+  hotglue flows list              List flows for a given environment
+  hotglue jobs list               List all jobs for a given flow
+  hotglue jobs download           Download jobroot files to the specified location
+  hotglue tenants list            List the tenants in the environment
+
+Options:
+  -k, --apikey              Your API Key                                  [string]
+  -e, --environment         Environment ID                                [string]
+  -f, --flow                Flow ID                                       [string]
+  -t, --tap                 Tap ID                                        [string]
+  -u, --tenant              Tenant ID                                     [string]
+  -j, --jobroot             A job root path                               [string]
+  -d, --downwloadTo         The target download location                  [string]
+  -s, --sourceFolder        The source directory                          [string]
+  -o, --overwrite           Overwrite existing files?                     [boolean]
+
+  -v, --version             Show version number                           [boolean]
+  -h, --help                Show help                                     [boolean]
+```
+
+## Configuration
+
+The [hotglue](https://hotglue.xyz) CLI derives its configuration from two places: the User/Profile settings and the Project settings.
+
+The User settings are stored in a `config.yaml` file located at `$HOME/.hotglue/`. At the moment, it is only used to store your `apikey`.
+
+The Project settings are stored in a hotglue rc-like file and can be created anywhere in your project's folder structure. A Project settings file can hold any of the CLI's parameters (including overriding the apikey, even though we do not recommend it) is used in order to define configuration values specific for that context
+
+### Project configuration files
+
+The configuration files can use any of the following names:
+`hotglue.yaml`, `.hotglue.yaml`, `.hotgluerc.yaml`, `.hotgluerc.yml`, `.hotgluerc`, `.hotgluerc.json`, `.hotgluerc.js`
+
+As the extensions suggest, the files support either yaml, json or js syntax (in case of JS, the config object has to be exported). However, we recommend using the yaml format.
+
+### Additional info
+
+- When deploying files, symbolic links are not followed.
+
+## Documentation
+
+- [hotglue CLI Documentation](https://docs.hotglue.xyz/docs/cli-overview)
+- [hotglue General Documentation](https://docs.hotglue.xyz)

package/lib/commands/config/index.js

@@ -0,0 +1,15 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.config = void 0;
+
+var set = _interopRequireWildcard(require("./set.js"));
+
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+
+const config = [set];
+exports.config = config;

package/lib/commands/config/set.js

@@ -0,0 +1,82 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.handler = exports.desc = exports.command = exports.builder = void 0;
+
+var _debug = _interopRequireDefault(require("../../helpers/debug.js"));
+
+var _yaml = _interopRequireDefault(require("yaml"));
+
+var _cliTable = _interopRequireDefault(require("cli-table"));
+
+var _config = require("../../helpers/config.js");
+
+var _print = require("../../helpers/print.js");
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+const debug = (0, _debug.default)('commands:config:set');
+const command = 'set <setting> <value>';
+exports.command = command;
+const desc = 'Set configuration key-value pairs';
+exports.desc = desc;
+
+const builder = async yargs => {
+  debug('builder', command, yargs);
+  yargs.option('setting', {
+    describe: 'The configuration parameter you wish to set',
+    type: 'string'
+  }).option('value', {
+    describe: "The configuration parameter's value",
+    type: 'string'
+  });
+  return yargs;
+};
+
+exports.builder = builder;
+
+const handler = async argv => {
+  debug('handler', command, argv);
+  const {
+    json,
+    setting,
+    value
+  } = argv;
+  const allowedSettings = ['apikey'];
+
+  if (!allowedSettings.includes(setting)) {
+    json ? (0, _print.printJSON)({
+      status: 'error',
+      error: 'Invalid settings parameter'
+    }) : console.error('Invalid settings parameter'); // throw new Error('Invalid settings parameter');
+
+    return;
+  }
+
+  try {
+    const profileConfig = await (0, _config.getProfileConfig)();
+    if (profileConfig && profileConfig.config) !json && (0, _print.pp)('Updating profile config file...');else !json && (0, _print.pp)('Creating profile config file...');
+    const data = profileConfig && profileConfig.config ? { ...profileConfig.config,
+      [setting]: value
+    } : {
+      [setting]: value
+    };
+
+    const yamlData = _yaml.default.stringify(data);
+
+    const filePath = profileConfig && profileConfig.filepath ? profileConfig.filepath : (0, _config.getProfileConfigFilePath)();
+    await (0, _config.writeConfigFile)(filePath, yamlData);
+    json ? (0, _print.printJSON)({
+      status: 'success'
+    }) : (0, _print.pp)('Done').pr();
+  } catch (err) {
+    json ? (0, _print.printJSON)({
+      status: 'error',
+      error: err
+    }) : console.log(err);
+  }
+};
+
+exports.handler = handler;

package/lib/commands/config.js

@@ -0,0 +1,71 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.handler = exports.desc = exports.command = exports.builder = void 0;
+
+var _path = _interopRequireDefault(require("path"));
+
+var _debug = _interopRequireDefault(require("../helpers/debug.js"));
+
+var _yaml = _interopRequireDefault(require("yaml"));
+
+var _cliTable = _interopRequireDefault(require("cli-table"));
+
+var _config = require("../helpers/config.js");
+
+var _print = require("../helpers/print.js");
+
+var _index = require("./config/index.js");
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+const debug = (0, _debug.default)('commands:config');
+const command = 'config [action]';
+exports.command = command;
+const desc = 'Configure your hotglue CLI';
+exports.desc = desc;
+
+const builder = async function (yargs) {
+  debug('builder', command);
+  return yargs.command(_index.config);
+};
+
+exports.builder = builder;
+
+const handler = async function (argv) {
+  debug('handler', command, argv);
+  const {
+    action
+  } = argv; // implement using yarg default command
+
+  if (!action) {
+    try {
+      const profileConfig = await (0, _config.getProfileConfig)();
+
+      if (!profileConfig.config || Object.keys(profileConfig.config).length === 0) {
+        (0, _print.pr)('No profile configuration found. Run config set to configure the hotglue CLI.');
+      }
+
+      const table = new _cliTable.default({
+        head: ['Setting', 'Value', 'Config File', 'Type']
+      });
+      if (profileConfig.config) Object.entries(profileConfig.config).forEach(([k, v]) => table.push([k, v, profileConfig.filepath, `Profile`]));
+      const projectConfig = await (0, _config.getProjectConfig)();
+      if (projectConfig.config) Object.entries(projectConfig.config).forEach(([k, v]) => table.push([k, v, _path.default.relative(process.cwd(), projectConfig.filepath), 'Project']));
+
+      if (table.length > 0) {
+        // pr('CURRENT CONTEXT SETTINGS');
+        console.log(table.toString()); // console.log(YAML.stringify(fileConfig.config));
+      } // pr('The configuration file is empty. Run config set to configure the hotglue CLI.');
+
+    } catch (err) {
+      // log elsewhere
+      console.log(err);
+      throw err;
+    }
+  }
+};
+
+exports.handler = handler;

package/lib/commands/env/deploy.js

@@ -0,0 +1,162 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.handler = exports.desc = exports.command = exports.builder = void 0;
+
+var _path = _interopRequireDefault(require("path"));
+
+var _promises = require("fs/promises");
+
+var _debug = _interopRequireDefault(require("../../helpers/debug.js"));
+
+var _ora = _interopRequireDefault(require("ora"));
+
+var _axios = _interopRequireDefault(require("axios"));
+
+var _awsSdk = _interopRequireDefault(require("aws-sdk"));
+
+var _cliTable = _interopRequireDefault(require("cli-table"));
+
+var _descriptions = _interopRequireDefault(require("../../helpers/descriptions.js"));
+
+var _utils = require("../../helpers/utils.js");
+
+var _print = require("../../helpers/print.js");
+
+var _api = require("../../helpers/api.js");
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+// import logUpdate from 'log-update';
+const debug = (0, _debug.default)('commands:env:deploy');
+const command = 'deploy';
+exports.command = command;
+const desc = 'Deploy Environment settings';
+exports.desc = desc;
+
+const builder = async yargs => {
+  debug('builder', command);
+  return yargs.option('sourceFolder', _descriptions.default.options['sourceFolder'].config).demandOption('sourceFolder', _descriptions.default.options['sourceFolder'].demandText);
+};
+
+exports.builder = builder;
+
+const handler = async argv => {
+  debug('handler', command, argv);
+  const {
+    hg,
+    json,
+    apikey,
+    env,
+    sourceFolder
+  } = argv;
+  const {
+    clientApiBaseUri
+  } = hg;
+
+  const folderPrefix = _path.default.resolve(process.cwd(), sourceFolder);
+
+  let message;
+  let spinner = (0, _ora.default)();
+
+  try {
+    // 1. build list of deployable files
+    message = (0, _print.themed)(`Scanning ${(0, _print.themed)(folderPrefix, 'info')} for deployable files`);
+    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
+    const deployableFiles = await (0, _utils.getFolderFiles)(folderPrefix, {
+      filter: {
+        pattern: '((**/requirements.txt)|(**/availableSources.json)|(**/availableTargets.json)|(**/availableConnectors.json))'
+      }
+    });
+
+    if (deployableFiles.length === 0) {
+      if (json) {
+        (0, _print.printJSON)({
+          status: 'error',
+          error: 'There are no files to deploy at the specified location!'
+        });
+      } else {
+        spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
+        (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)('There are no files to deploy at the specified location!')}`, 'secondary'));
+      }
+
+      return;
+    }
+
+    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // 2. get STS credentials
+
+    message = (0, _print.themed)(`Verifying user and authorizing`);
+    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
+    const {
+      accessKeyId,
+      secretAccessKey,
+      sessionToken
+    } = await (0, _api.genCredentialsOnClientApi)({
+      debug,
+      baseUri: clientApiBaseUri,
+      task: 'env-deploy',
+      env,
+      apikey
+    });
+    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // create authenticated S3 instance
+
+    const s3 = new _awsSdk.default.S3({
+      accessKeyId,
+      secretAccessKey,
+      sessionToken
+    }); // 3. upload new files
+
+    message = (0, _print.themed)(`Deploying environment files`);
+    !json && spinner.info((0, _print.themed)(`Info: ${message}.`, 'secondary'));
+    const rootKey = `config/`;
+    const table = new _cliTable.default({
+      head: ['File', 'Status']
+    });
+
+    for await (const file of deployableFiles) {
+      const filename = _path.default.basename(file);
+
+      message = (0, _print.themed)(`Pushing file: ${(0, _print.themed)(filename, 'info')}`);
+      !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
+      const readBuffer = await (0, _promises.readFile)(file);
+      const params = {
+        Bucket: env,
+        Key: `${rootKey}${filename}`,
+        Body: readBuffer
+      };
+      const res = await s3.putObject(params).promise();
+      !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
+      debug('s3-put-res', res);
+      table.push([filename, 'Deployed']); // handling redis change
+
+      await (0, _api.resetAvailableEntities)({
+        debug,
+        baseUri: clientApiBaseUri,
+        env,
+        apiKey: apikey
+      });
+    }
+
+    if (json) {
+      (0, _print.printJSON)({
+        status: 'success',
+        deployedFiles: deployableFiles
+      });
+    } else {
+      // print results
+      (0, _print.cl)(table.toString());
+    }
+  } catch (err) {
+    !json && spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
+    (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
+    debug(err);
+
+    if (err && err.response && err.response.data) {
+      debug('response', err.response.data);
+    }
+  }
+};
+
+exports.handler = handler;

package/lib/commands/env/download.js

@@ -0,0 +1,169 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.handler = exports.desc = exports.command = exports.builder = void 0;
+
+var _path = _interopRequireDefault(require("path"));
+
+var _promises = require("fs/promises");
+
+var _debug = _interopRequireDefault(require("../../helpers/debug.js"));
+
+var _ora = _interopRequireDefault(require("ora"));
+
+var _axios = _interopRequireDefault(require("axios"));
+
+var _cliTable = _interopRequireDefault(require("cli-table"));
+
+var _awsSdk = _interopRequireDefault(require("aws-sdk"));
+
+var _descriptions = _interopRequireDefault(require("../../helpers/descriptions.js"));
+
+var _print = require("../../helpers/print.js");
+
+var _utils = require("../../helpers/utils.js");
+
+var _api = require("../../helpers/api.js");
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+const debug = (0, _debug.default)('commands:env:download');
+const command = 'download';
+exports.command = command;
+const desc = 'Download Environment settings';
+exports.desc = desc;
+
+const builder = async yargs => {
+  debug('builder', command); // hotglue environment download -e <env> -o -d downloads
+
+  return yargs.option('downloadTo', _descriptions.default.options['downloadTo'].config); // .option('overwrite', descriptions.options['overwrite'].config);
+};
+
+exports.builder = builder;
+
+const handler = async argv => {
+  debug('handler', command, argv);
+  const {
+    hg,
+    json,
+    apikey,
+    env,
+    downloadTo
+  } = argv;
+  const {
+    clientApiBaseUri
+  } = hg;
+
+  const folderPrefix = _path.default.resolve(process.cwd(), downloadTo);
+
+  let message;
+  let spinner = (0, _ora.default)();
+
+  try {
+    // 1. get STS credentials
+    message = (0, _print.themed)(`Verifying user and authorizing`);
+    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
+    const {
+      accessKeyId,
+      secretAccessKey,
+      sessionToken
+    } = await (0, _api.genCredentialsOnClientApi)({
+      debug,
+      baseUri: clientApiBaseUri,
+      task: 'env-download',
+      env,
+      apikey
+    });
+    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // create authenticated S3 instance
+
+    const s3 = new _awsSdk.default.S3({
+      accessKeyId,
+      secretAccessKey,
+      sessionToken
+    }); // 2. get list of files from AWS bucket
+
+    message = (0, _print.themed)(`Scanning environment ${(0, _print.themed)(env, 'info')}`);
+    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
+    const params = {
+      Bucket: env,
+      Prefix: `config/`
+    };
+    const {
+      Contents: s3Files
+    } = await s3.listObjectsV2(params).promise();
+    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
+    debug('s3-files', s3Files); // 3. filter only downloadable files
+
+    const filesToDownload = s3Files ? (0, _utils.filterFiles)(s3Files.map(item => item.Key), {
+      pattern: '((*/requirements.txt)|(*/availableSources.json)|(*/availableTargets.json)|(*/availableConnectors.json))'
+    }) : [];
+
+    if (!filesToDownload || filesToDownload.length === 0) {
+      json ? (0, _print.printJSON)({
+        status: 'success',
+        downloadedFiles: []
+      }) : spinner.warn((0, _print.themed)(`Warning: ${(0, _print.themed)('Nothing to download!')}`, 'secondary'));
+      return;
+    } // 4. download environment config files
+
+
+    message = (0, _print.themed)(`Downloading to ${(0, _print.themed)(folderPrefix, 'info')}`);
+    !json && spinner.info((0, _print.themed)(`Info: ${message}.`, 'secondary'));
+    const table = new _cliTable.default({
+      head: ['File', 'Status']
+    });
+
+    for await (const item of filesToDownload) {
+      // create destination folder
+      const filename = _path.default.basename(item);
+
+      const fullPath = _path.default.resolve(folderPrefix, filename);
+
+      debug('local-file', fullPath); // ensure destination folder exists
+
+      await (0, _promises.mkdir)(_path.default.dirname(fullPath), {
+        recursive: true
+      }); // download s3 object into destination folder
+
+      message = (0, _print.themed)(`Downloading file: ${(0, _print.themed)(filename, 'info')}`);
+      !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
+      const fileStream = s3.getObject({
+        Bucket: env,
+        Key: item
+      }).createReadStream();
+      await (0, _promises.writeFile)(fullPath, await (0, _utils.streamToString)(fileStream));
+      !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // debug('downloaded', env + item);
+
+      table.push([filename, 'Downloaded']);
+    }
+
+    if (json) {
+      (0, _print.printJSON)({
+        status: 'success',
+        downloadedFiles: filesToDownload
+      });
+    } else {
+      // print results
+      (0, _print.cl)(table.toString());
+    }
+  } catch (err) {
+    if (json) {
+      (0, _print.printJSON)({
+        status: 'error',
+        error: err
+      });
+    } else {
+      spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
+      (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
+      debug(err);
+
+      if (err && err.response && err.response.data) {
+        debug('response', err.response.data);
+      }
+    }
+  }
+};
+
+exports.handler = handler;

package/lib/commands/env/index.js

@@ -0,0 +1,17 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.env = void 0;
+
+var deploy = _interopRequireWildcard(require("./deploy.js"));
+
+var download = _interopRequireWildcard(require("./download.js"));
+
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+
+const env = [deploy, download];
+exports.env = env;

package/lib/commands/env.js

@@ -0,0 +1,34 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.handler = exports.desc = exports.command = exports.builder = void 0;
+
+var _debug = _interopRequireDefault(require("../helpers/debug.js"));
+
+var _baseBuilder = require("../helpers/baseBuilder.js");
+
+var _index = require("./env/index.js");
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+const debug = (0, _debug.default)('commands:env');
+const command = 'env <action>';
+exports.command = command;
+const desc = 'Manage environment settings';
+exports.desc = desc;
+
+const builder = async function (yargs) {
+  debug('builder', command);
+  const base = await (0, _baseBuilder.baseApiBuilder)(yargs);
+  return base.command(_index.env);
+};
+
+exports.builder = builder;
+
+const handler = async function (argv) {
+  debug('handler', command, argv);
+};
+
+exports.handler = handler;