clever-tools 2.11.0 → 3.0.0

package/CHANGELOG.md

@@ -1,6 +1,25 @@
 # clever-tools changelog
 
-## Unrelease (????-??-??)
+## Unreleased (????-??-??)
+
+...
+
+## 3.0.0 (2023-10-19)
+
+### ⚠ BREAKING CHANGES
+
+* Move from Node.js v12.22.8 to v18.5.0
+
+### Features
+
+* add config file and auth source to `clever diag`
+* add shell to `clever diag`
+* improve `clever diag`, display (color and details) for oAuth token and user ID
+* `clever deploy`: use new logs API (faster, longer, order)
+* `clever restart`: use new logs API (faster, longer, order)
+* `clever logs`: use new logs API (faster, longer, order), only for applications for now
+
+## 2.11.0 (2023-07-25)
 
 * skip preorder step on addon creation
 * add `clever addon env` command

package/bin/clever.js

@@ -506,6 +506,7 @@ function run () {
       description: 'A WireGuard® private key',
     }),
     jsonFormat: cliparse.flag('json', { aliases: ['j'], description: 'Show result in JSON format' }),
+    humanJsonOutputFormat: getOutputFormatOption(),
     tag: cliparse.option('tag', {
       required: true,
       metavar: 'tag',
@@ -626,7 +627,7 @@ function run () {
   const appCreateCommand = cliparse.command('create', {
     description: 'Create a Clever Cloud application',
     args: [args.appNameCreation],
-    options: [opts.instanceType, opts.orgaIdOrName, opts.aliasCreation, opts.region, opts.github],
+    options: [opts.instanceType, opts.orgaIdOrName, opts.aliasCreation, opts.region, opts.github, opts.humanJsonOutputFormat],
   }, create('create'));
 
   // DELETE COMMAND
@@ -828,11 +829,13 @@ function run () {
   });
   // peer category - end
 
+  // eslint-disable-next-line no-unused-vars
   const networkGroupsCommand = cliparse.command('networkgroups', {
     description: 'List Network Group commands',
     options: [opts.orgaIdOrName, opts.alias],
     commands: [networkGroupsListCommand, networkGroupsCreateCommand, networkGroupsDeleteCommand, networkGroupsMembersCategoryCommand, networkGroupsPeersCategoryCommand],
   });
+  // eslint-disable-next-line no-unused-vars
   const ngCommand = cliparse.command('ng', {
     description: `Alias for ${Formatter.formatCommand('clever networkgroups')}`,
     options: [opts.orgaIdOrName, opts.alias],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clever-tools",
-  "version": "2.11.0",
+  "version": "3.0.0",
   "description": "Command Line Interface for Clever Cloud.",
   "main": "bin/clever.js",
   "keywords": [
@@ -9,9 +9,9 @@
     "clever cloud"
   ],
   "engines": {
-    "node": ">=12"
+    "node": ">=18"
   },
-  "pkg-node-version": "12",
+  "pkg-node-version": "18",
   "author": "Clever Cloud <ci@clever-cloud.com>",
   "license": "Apache-2.0",
   "bin": {
@@ -25,7 +25,7 @@
     "scripts/*.sh"
   ],
   "dependencies": {
-    "@clevercloud/client": "7.9.0",
+    "@clevercloud/client": "^8.0.1",
     "clf-date": "^0.2.0",
     "cliparse": "^0.3.3",
     "colors": "1.4.0",
@@ -64,7 +64,7 @@
     "grunt-http": "^2.3.3",
     "grunt-mocha-test": "^0.13.3",
     "mocha": "^8.4.0",
-    "pkg": "^5.2.1",
+    "pkg": "^5.8.1",
     "semver": "^7.3.5"
   },
   "scripts": {
@@ -87,7 +87,6 @@
     ]
   },
   "volta": {
-    "node": "12.22.8",
-    "npm": "6.14.15"
+    "node": "18.5.0"
   }
 }

package/src/commands/create.js

@@ -3,17 +3,28 @@
 const Application = require('../models/application.js');
 const AppConfig = require('../models/app_configuration.js');
 const Logger = require('../logger.js');
+const { toNameEqualsValueString } = require('@clevercloud/client/cjs/utils/env-vars.js');
 
 async function create (params) {
   const { type: typeName } = params.options;
   const [name] = params.args;
-  const { org: orgaIdOrName, alias, region, github: githubOwnerRepo } = params.options;
+  const { org: orgaIdOrName, alias, region, github: githubOwnerRepo, format } = params.options;
   const github = getGithubDetails(githubOwnerRepo);
 
   const app = await Application.create(name, typeName, region, orgaIdOrName, github);
   await AppConfig.addLinkedApplication(app, alias);
 
-  Logger.println('Your application has been successfully created!');
+  switch (format) {
+
+    case 'json': {
+      console.log(JSON.stringify(app, null, 2));
+      break;
+    }
+
+    case 'human':
+    default:
+      Logger.println('Your application has been successfully created!');
+  }
 };
 
 function getGithubDetails (githubOwnerRepo) {

package/src/commands/curl.js

@@ -3,7 +3,7 @@
 const { parseCurlCommand } = require('curlconverter/util.js');
 const { spawn } = require('child_process');
 const { loadOAuthConf, conf } = require('../models/configuration.js');
-const { addOauthHeader } = require('@clevercloud/client/cjs/oauth.node.js');
+const { addOauthHeader } = require('@clevercloud/client/cjs/oauth.js');
 
 async function loadTokens () {
   const tokens = await loadOAuthConf();

package/src/commands/deploy.js

@@ -5,7 +5,7 @@ const colors = require('colors/safe');
 const AppConfig = require('../models/app_configuration.js');
 const Application = require('../models/application.js');
 const git = require('../models/git.js');
-const Log = require('../models/log.js');
+const Log = require('../models/log-v4.js');
 const Logger = require('../logger.js');
 const { getAllDeployments } = require('@clevercloud/client/cjs/api/v2/application.js');
 const { sendToApi } = require('../models/send-to-api.js');

package/src/commands/diag.js

@@ -8,12 +8,12 @@ const colors = require('colors/safe');
 const Logger = require('../logger.js');
 const pkg = require('../../package.json');
 const User = require('../models/user.js');
-const { loadOAuthConf } = require('../models/configuration.js');
+const { conf, loadOAuthConf } = require('../models/configuration.js');
 
 async function diag () {
 
-  const userId = await User.getCurrentId().catch(() => 'Not connected');
-  const oauthToken = await loadOAuthConf().then((o) => o.token).catch(() => 'Not connected');
+  const userId = await User.getCurrentId().catch(() => null);
+  const authDetails = await loadOAuthConf();
 
   Logger.println('clever-tools  ' + colors.green(pkg.version));
   Logger.println('Node.js       ' + colors.green(process.version));
@@ -28,12 +28,30 @@ async function diag () {
     Logger.println('Linux         ' + colors.green(linuxInfos));
   }
 
+  Logger.println('Shell         ' + colors.green(process.env.SHELL));
+
   const isPackaged = (process.pkg != null);
   Logger.println('Packaged      ' + colors.green(isPackaged));
   Logger.println('Exec path     ' + colors.green(process.execPath));
-
-  Logger.println('User id       ' + colors.green(userId || 'Not connected'));
-  Logger.println('oAuth token   ' + colors.green(oauthToken));
+  Logger.println('Config file   ' + colors.green(conf.CONFIGURATION_FILE));
+  Logger.println('Auth source   ' + colors.green(authDetails.source));
+
+  const oauthToken = (authDetails.token != null)
+    ? colors.green(authDetails.token)
+    : colors.red('(none)');
+  Logger.println('oAuth token   ' + oauthToken);
+
+  if (authDetails.token != null) {
+    if (userId != null) {
+      Logger.println('User ID       ' + colors.green(userId));
+    }
+    else {
+      Logger.println('User ID       ' + colors.red('Authentication failed'));
+    }
+  }
+  else {
+    Logger.println('User ID       ' + colors.red('Not connected'));
+  }
 }
 
 module.exports = { diag };

package/src/commands/logs.js

@@ -1,20 +1,31 @@
 'use strict';
 
 const AppConfig = require('../models/app_configuration.js');
-const Log = require('../models/log.js');
+const LogV2 = require('../models/log.js');
+const Log = require('../models/log-v4.js');
 const Logger = require('../logger.js');
+const { Deferred } = require('../models/utils.js');
 
 async function appLogs (params) {
   const { alias, after: since, before: until, search, 'deployment-id': deploymentId } = params.options;
-  const { addon: addonId } = params.options;
 
   // ignore --search ""
   const filter = (search !== '') ? search : null;
-  const appAddonId = addonId || await AppConfig.getAppDetails({ alias }).then(({ appId }) => appId);
+
+  const { appId, ownerId } = await AppConfig.getAppDetails({ alias });
 
   Logger.println('Waiting for application logs…');
 
-  return Log.displayLogs({ appAddonId, since, until, filter, deploymentId });
+  // TODO: drop when addons are migrated to the v4 API
+  if (params.addon) {
+    const { addon: addonId } = params.options;
+    const appAddonId = addonId || await AppConfig.getAppDetails({ alias }).then(({ appId }) => appId);
+    return LogV2.displayLogs({ appAddonId, since, until, filter, deploymentId });
+  }
+
+  const deferred = new Deferred();
+  await Log.displayLogs({ ownerId, appId, since, until, filter, deploymentId, deferred });
+  return deferred.promise;
 }
 
 module.exports = { appLogs };

package/src/commands/restart.js

@@ -5,7 +5,7 @@ const colors = require('colors/safe');
 const AppConfig = require('../models/app_configuration.js');
 const Application = require('../models/application.js');
 const git = require('../models/git.js');
-const Log = require('../models/log.js');
+const Log = require('../models/log-v4.js');
 const Logger = require('../logger.js');
 
 // Once the API call to redeploy() has been triggerred successfully,
@@ -24,9 +24,19 @@ async function restart (params) {
     Logger.println(`Restarting ${appName} on commit ${colors.green(commitId)}${cacheSuffix}`);
   }
 
+  // This should be handled by the API when a deployment ID is set but we'll do this for now
+  const redeployDate = new Date();
+
   const redeploy = await Application.redeploy(ownerId, appId, fullCommitId, withoutCache);
 
-  return Log.watchDeploymentAndDisplayLogs({ ownerId, appId, deploymentId: redeploy.deploymentId, quiet, follow });
+  return Log.watchDeploymentAndDisplayLogs({
+    ownerId,
+    appId,
+    deploymentId: redeploy.deploymentId,
+    quiet,
+    follow,
+    redeployDate,
+  });
 }
 
 module.exports = { restart };

package/src/models/configuration.js

@@ -26,6 +26,7 @@ async function loadOAuthConf () {
   Logger.debug('Load configuration from environment variables');
   if (process.env.CLEVER_TOKEN != null && process.env.CLEVER_SECRET != null) {
     return {
+      source: 'environment variables',
       token: process.env.CLEVER_TOKEN,
       secret: process.env.CLEVER_SECRET,
     };
@@ -33,11 +34,18 @@ async function loadOAuthConf () {
   Logger.debug('Load configuration from ' + conf.CONFIGURATION_FILE);
   try {
     const rawFile = await fs.readFile(conf.CONFIGURATION_FILE);
-    return JSON.parse(rawFile);
+    const { token, secret } = JSON.parse(rawFile);
+    return {
+      source: 'configuration file',
+      token,
+      secret,
+    };
   }
   catch (error) {
     Logger.info(`Cannot load configuration from ${conf.CONFIGURATION_FILE}\n${error.message}`);
-    return {};
+    return {
+      source: 'none',
+    };
   }
 }
 

package/src/models/format-ng-table.js

@@ -78,7 +78,7 @@ const peersTableColumnLengths = [
   36, /* label length */
 ];
 const formatPeersTable = formatNgTable(peersTableColumnLengths);
-function formatPeersLine(peer) {
+function formatPeersLine (peer) {
   const ip = (peer.endpoint.type === 'ServerEndpoint') ? peer.endpoint.ng_term.host : peer.endpoint.ng_ip;
   return formatPeersTable([
     [

package/src/models/log-v4.js

@@ -0,0 +1,136 @@
+const { getHostAndTokens } = require('./send-to-api.js');
+const colors = require('colors/safe');
+const { Deferred } = require('./utils.js');
+const Logger = require('../logger.js');
+const { waitForDeploymentEnd, waitForDeploymentStart } = require('./deployments.js');
+const { ApplicationLogStream } = require('@clevercloud/client/cjs/streams/application-logs.js');
+
+// 2000 logs per 100ms maximum
+const THROTTLE_ELEMENTS = 2000;
+const THROTTLE_PER_IN_MILLISECONDS = 100;
+
+async function displayLogs (params) {
+
+  const deferred = params.deferred || new Deferred();
+  const { apiHost, tokens } = await getHostAndTokens();
+  const { ownerId, appId, filter, since, until, deploymentId } = params;
+
+  const logStream = new ApplicationLogStream({
+    apiHost,
+    tokens,
+    ownerId,
+    appId,
+    since,
+    until,
+    deploymentId,
+    filter,
+    throttleElements: THROTTLE_ELEMENTS,
+    throttlePerInMilliseconds: THROTTLE_PER_IN_MILLISECONDS,
+  });
+
+  // Properly close the stream
+  process.once('SIGINT', (signal) => logStream.close(signal));
+
+  logStream
+    .on('open', (event) => {
+      Logger.debug(`stream opened! ${JSON.stringify({ appId, filter, deploymentId })}`);
+    })
+    .on('error', (event) => {
+      Logger.error(`an error occured: ${event.detail}`);
+    })
+    .onLog((log) => {
+      Logger.println(formatLogLine(log));
+    });
+
+  // start() is blocking until end of stream
+  logStream.start()
+    .then((reason) => deferred.resolve())
+    .catch((error) => deferred.reject(error));
+
+  return logStream;
+}
+
+async function watchDeploymentAndDisplayLogs (options) {
+
+  const {
+    ownerId,
+    appId,
+    deploymentId,
+    commitId,
+    knownDeployments,
+    quiet,
+    follow,
+    redeployDate,
+  } = options;
+
+  Logger.println('Waiting for deployment to start…');
+  const deployment = await waitForDeploymentStart({ ownerId, appId, deploymentId, commitId, knownDeployments });
+  Logger.println(colors.bold.blue(`Deployment started (${deployment.uuid})`));
+
+  const deferred = new Deferred();
+  let logsStream;
+
+  if (!quiet) {
+    // About the deferred…
+    // If displayLogs() throws an error,
+    // the async function we're in (watchDeploymentAndDisplayLogs) will stop here and the error will be passed to the parent.
+    // displayLogs() defines callback listeners so if it catches error in those callbacks,
+    // it has no proper way to bubble up the error here.
+    // Using the deferred enables this.
+    logsStream = await displayLogs({ ownerId, appId, deploymentId: deployment.uuid, since: redeployDate, deferred });
+  }
+
+  Logger.println('Waiting for application logs…');
+
+  // Wait for deployment end (or an error thrown by logs with the deferred)
+  const deploymentEnded = await Promise.race([
+    waitForDeploymentEnd({ ownerId, appId, deploymentId: deployment.uuid }),
+    deferred.promise,
+  ]);
+
+  if (!quiet && !follow) {
+    logsStream.close(quiet ? 'quiet' : 'follow');
+  }
+
+  if (deploymentEnded.state === 'OK') {
+    Logger.println(colors.bold.green('Deployment successful'));
+  }
+  else if (deploymentEnded.state === 'CANCELLED') {
+    throw new Error('Deployment was cancelled. Please check the activity');
+  }
+  else {
+    throw new Error('Deployment failed. Please check the logs');
+  }
+}
+
+function formatLogLine (log) {
+  const { date, message } = log;
+  if (isDeploymentSuccessMessage(log)) {
+    return `${date.toISOString()}: ${colors.bold.green(message)}`;
+  }
+  else if (isDeploymentFailedMessage(log)) {
+    return `${date.toISOString()}: ${colors.bold.red(message)}`;
+  }
+  else if (isBuildSucessMessage(log)) {
+    return `${date.toISOString()}: ${colors.bold.blue(message)}`;
+  }
+  return `${date.toISOString()}: ${message}`;
+}
+
+function isCleverMessage (log) {
+  return log.service !== 'bas-deploy.service';
+};
+
+function isDeploymentSuccessMessage (log) {
+  return isCleverMessage(log) && log.message.toLowerCase().startsWith('successfully deployed in');
+};
+
+function isDeploymentFailedMessage (log) {
+  return isCleverMessage(log) && log.message.toLowerCase().startsWith('deploy failed in');
+};
+
+function isBuildSucessMessage (log) {
+  return isCleverMessage(log) && log.message.toLowerCase().startsWith('build succeeded in');
+};
+
+module.exports = { displayLogs, watchDeploymentAndDisplayLogs };

package/src/models/send-to-api.js

@@ -1,11 +1,11 @@
 'use strict';
 
 const Logger = require('../logger.js');
-const { addOauthHeader } = require('@clevercloud/client/cjs/oauth.node.js');
+const { addOauthHeader } = require('@clevercloud/client/cjs/oauth.js');
 const { conf, loadOAuthConf } = require('../models/configuration.js');
 const { execWarpscript } = require('@clevercloud/client/cjs/request-warp10.superagent.js');
 const { prefixUrl } = require('@clevercloud/client/cjs/prefix-url.js');
-const { request } = require('@clevercloud/client/cjs/request.superagent.js');
+const { request } = require('@clevercloud/client/cjs/request.fetch.js');
 
 async function loadTokens () {
   const tokens = await loadOAuthConf();

package/src/parsers.js

@@ -2,7 +2,6 @@
 
 const cliparse = require('cliparse');
 
-const AccessLogs = require('./models/accesslogs.js');
 const Application = require('./models/application.js');
 
 function flavor (flavor) {