clever-tools 3.6.1 → 3.8.0

package/src/commands/accesslogs.js

@@ -1,53 +1,133 @@
 'use strict';
 
-const { getAccessLogsFromWarp10InBatches, getContinuousAccessLogsFromWarp10 } = require('@clevercloud/client/cjs/access-logs.js');
-const { getWarp10AccessLogsToken } = require('@clevercloud/client/cjs/api/v2/warp-10.js');
-const { ONE_HOUR_MICROS, ONE_SECOND_MICROS, toMicroTimestamp } = require('@clevercloud/client/cjs/utils/date.js');
-
-const Addon = require('../models/addon.js');
-const AppConfig = require('../models/app_configuration.js');
+const Application = require('../models/application.js');
 const Logger = require('../logger.js');
-const { getFormatter } = require('../models/accesslogs.js');
-const { sendToApi, sendToWarp10 } = require('../models/send-to-api.js');
+const { getHostAndTokens } = require('../models/send-to-api.js');
+const { ApplicationAccessLogStream } = require('@clevercloud/client/cjs/streams/access-logs.js');
+const { JsonArray } = require('../models/json-array.js');
+const colors = require('colors/safe');
+const formatTable = require('../format-table.js');
+const { truncateWithEllipsis } = require('../models/utils.js');
 
-const CONTINUOUS_DELAY = ONE_SECOND_MICROS * 5;
+// 2000 logs per 100ms maximum
+const THROTTLE_ELEMENTS = 2000;
+const THROTTLE_PER_IN_MILLISECONDS = 100;
+const CITY_MAX_LENGTH = 20;
 
 async function accessLogs (params) {
-  const { alias, format, before, after, addon: addonId, follow } = params.options;
 
-  const { ownerId, appId, realAddonId } = await getIds(addonId, alias);
-  const to = (before != null) ? toMicroTimestamp(before.toISOString()) : toMicroTimestamp();
-  const from = (after != null) ? toMicroTimestamp(after.toISOString()) : to - ONE_HOUR_MICROS;
-  const warpToken = await getWarp10AccessLogsToken({ orgaId: ownerId }).then(sendToApi);
+  // TODO: drop when add-ons are supported in API
+  if (params.options.addon) {
+    throw new Error('Access Logs are not available for add-ons yet');
+  }
+
+  const { apiHost, tokens } = await getHostAndTokens();
+  const { alias, app: appIdOrName, format, before: until, after: since } = params.options;
+  const { ownerId, appId } = await Application.resolveId(appIdOrName, alias);
+
+  const stream = new ApplicationAccessLogStream({
+    apiHost,
+    tokens,
+    ownerId,
+    appId,
+    since,
+    until,
+    throttleElements: THROTTLE_ELEMENTS,
+    throttlePerInMilliseconds: THROTTLE_PER_IN_MILLISECONDS,
+  });
+
+  if (format === 'human') {
+    Logger.warn(colors.yellow('/!\\ Access Logs feature is in Alpha testing phase'));
+  }
 
-  if (follow && (before != null || after != null)) {
-    Logger.warn('Access logs are displayed continuously with -f/--follow therefore --before and --after are ignored.');
+  if (format === 'json' && (!until)) {
+    throw new Error('JSON format only works with a limiting parameter such as `before`');
   }
 
-  const emitter = follow
-    ? getContinuousAccessLogsFromWarp10({ appId, realAddonId, warpToken, delay: CONTINUOUS_DELAY }, sendToWarp10)
-    : getAccessLogsFromWarp10InBatches({ appId, realAddonId, from, to, warpToken }, sendToWarp10);
+  // used for 'json' format
+  const jsonArray = new JsonArray();
 
-  const formatLogLine = getFormatter(format, addonId != null);
+  stream
+    .on('open', (event) => {
+      Logger.debug(colors.blue(`Logs stream (open) ${JSON.stringify({ appId })}`));
+      if (format === 'json') {
+        jsonArray.open();
+      }
+    })
+    .on('error', (event) => {
+      Logger.debug(colors.red(`Logs stream (error) ${event.error.message}`));
+    })
+    .onLog((log) => {
+      switch (format) {
+        case 'json':
+          jsonArray.push(log);
+          break;
+        case 'json-stream':
+          Logger.printJson(log);
+          break;
+        case 'human':
+        default:
+          // when the connection is cut too early, or for TCP redirections, we don't have HTTP section
+          if (log.http == null) {
+            break;
+          }
 
-  emitter.on('data', (data) => {
-    data.forEach((l) => Logger.println(formatLogLine(l)));
-  });
+          Logger.println(formatHuman(log));
+          break;
+      }
+    });
 
-  return new Promise((resolve, reject) => {
-    emitter.on('error', reject);
-  });
+  // Properly close the stream
+  process.once('SIGINT', (signal) => stream.close(signal));
+
+  const closeReason = await stream.start();
+
+  if (format === 'json') {
+    jsonArray.close();
+  }
+
+  Logger.debug(`stream closed: ${closeReason?.type}`);
+}
+
+function formatHuman (log) {
+  const { date, http, source } = log;
+  const country = source.countryCode ?? '(unknown)';
+  const hasSourceCity = source.city ?? '';
+
+  return row([[
+    colors.grey(date.toISOString(date)),
+    source.ip,
+    `${country}${hasSourceCity ? '/' + truncateWithEllipsis(CITY_MAX_LENGTH, source.city) : ''}`,
+    colorStatusCode(http.response.statusCode),
+    http.request.method.toString().padEnd(4, ' ') + ' ' + http.request.path,
+  ]]);
 }
 
-async function getIds (addonId, alias) {
-  if (addonId != null) {
-    const addon = await Addon.findById(addonId);
-    return {
-      ownerId: addon.orgaId,
-      realAddonId: addon.realId,
-    };
+const row = formatTable([
+  '2024-06-24T08:05:43.880Z',
+  '255.255.255.255',
+  // country / city
+  2 + 1 + CITY_MAX_LENGTH,
+  'XXX',
+  // longest method name
+  'OPTIONS',
+  // path
+]);
+
+function colorStatusCode (code) {
+  if (code >= 500) {
+    return colors.red(code);
+  }
+  if (code >= 400) {
+    return colors.yellow(code);
+  }
+  if (code >= 300) {
+    return colors.blue(code);
+  }
+  if (code >= 200) {
+    return colors.green(code);
   }
-  return AppConfig.getAppDetails({ alias });
+  return code;
 }
 
 module.exports = { accessLogs };

package/src/commands/activity.js

@@ -4,12 +4,12 @@ const colors = require('colors/safe');
 const moment = require('moment');
 
 const Activity = require('../models/activity.js');
-const AppConfig = require('../models/app_configuration.js');
 const formatTable = require('../format-table');
 const Logger = require('../logger.js');
 const { Deferred } = require('../models/utils.js');
 const { EventsStream } = require('@clevercloud/client/cjs/streams/events.node.js');
 const { getHostAndTokens } = require('../models/send-to-api.js');
+const Application = require('../models/application.js');
 
 function getColoredState (state, isLast) {
   if (state === 'OK') {
@@ -31,24 +31,37 @@ function getColoredState (state, isLast) {
 // We use examples of maximum width text to have a clean display
 const formatActivityTable = formatTable([
   moment().format(),
+  47,
   'IN PROGRESS',
   'downscale',
-  // a git commit id is 40 chars long
-  40,
+  // a git commit id is 8 chars long
+  8,
   0,
 ]);
 
+function convertEventToJson (event) {
+  return {
+    uuid: event.uuid,
+    date: event.date,
+    state: event.state,
+    action: event.action,
+    commit: event.commit,
+    cause: event.cause,
+  };
+}
+
 function formatActivityLine (event) {
   return formatActivityTable([
     [
       moment(event.date).format(),
+      event.uuid,
       getColoredState(event.state, event.isLast),
       event.action,
-      event.commit || 'not specified',
+      event?.commit?.substring(0, 8) ?? 'N/A',
       event.cause,
     ],
   ]);
-};
+}
 
 function isTemporaryEvent (ev) {
   if (ev == null) {
@@ -65,28 +78,65 @@ function clearPreviousLine () {
   }
 }
 
-function handleEvent (previousEvent, event) {
+function handleEvent (previousEvent, event, handler) {
   if (isTemporaryEvent(previousEvent)) {
-    clearPreviousLine();
+    handler.pop();
   }
 
-  const activityLine = formatActivityLine(event);
-  Logger.println(activityLine);
+  handler.print(event);
 
   return event;
 }
 
-function onEvent (previousEvent, newEvent) {
-  const { event, date, data: { state, action, commit, cause } } = newEvent;
+function onEvent (previousEvent, newEvent, handler) {
+  const { event, date, data: { uuid, state, action, commit, cause } } = newEvent;
   if (event !== 'DEPLOYMENT_ACTION_BEGIN' && event !== 'DEPLOYMENT_ACTION_END') {
     return previousEvent;
   }
-  return handleEvent(previousEvent, { date, state, action, commit, cause, isLast: true });
+  return handleEvent(previousEvent, { date, uuid, state, action, commit, cause, isLast: true }, handler);
+}
+
+function getEventHandler (format, follow) {
+  switch (format) {
+    case 'json': {
+      if (follow) {
+        throw new Error('The `follow` option and "json" format are not compatible. Use "json-stream" format instead.');
+      }
+      else {
+        const buf = [];
+
+        return {
+          print: (event) => buf.push(convertEventToJson(event)),
+          pop: () => buf.pop(),
+          end: () => {
+            Logger.printJson(buf);
+          },
+        };
+      }
+    }
+    case 'json-stream': {
+      return {
+        print: (event) => {
+          Logger.println(JSON.stringify(convertEventToJson(event)));
+        },
+        pop: clearPreviousLine,
+        end: () => {},
+      };
+    }
+    case 'human':
+    default: {
+      return {
+        print: (event) => Logger.println(formatActivityLine(event)),
+        pop: clearPreviousLine,
+        end: () => {},
+      };
+    }
+  }
 }
 
 async function activity (params) {
-  const { alias, 'show-all': showAll, follow } = params.options;
-  const { ownerId, appId } = await AppConfig.getAppDetails({ alias });
+  const { alias, app: appIdOrName, 'show-all': showAll, follow, format } = params.options;
+  const { ownerId, appId } = await Application.resolveId(appIdOrName, alias);
   const events = await Activity.list(ownerId, appId, showAll);
   const reversedArrayWithIndex = events
     .reverse()
@@ -94,9 +144,13 @@ async function activity (params) {
       const isLast = index === all.length - 1;
       return ({ ...event, isLast });
     });
-  let lastEvent = reversedArrayWithIndex.reduce(handleEvent, {});
+
+  const handler = getEventHandler(format, follow);
+
+  let lastEvent = reversedArrayWithIndex.reduce((previousEvent, newEvent) => handleEvent(previousEvent, newEvent, handler), {});
 
   if (!follow) {
+    handler.end();
     return lastEvent;
   }
 
@@ -108,7 +162,7 @@ async function activity (params) {
   eventsStream
     .on('open', () => Logger.debug('WS for events (open) ' + JSON.stringify({ appId })))
     .on('event', (event) => {
-      lastEvent = onEvent(lastEvent, event);
+      lastEvent = onEvent(lastEvent, event, handler);
       return lastEvent;
     })
     .on('ping', () => Logger.debug('WS for events (ping)'))

package/src/commands/addon.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const _ = require('lodash');
 const colors = require('colors/safe');
 
 const Addon = require('../models/addon.js');
@@ -15,22 +14,6 @@ const { sendToApi } = require('../models/send-to-api.js');
 const { toNameEqualsValueString } = require('@clevercloud/client/cjs/utils/env-vars.js');
 const { resolveAddonId } = require('../models/ids-resolver.js');
 
-function getPlan (inputPlan, providerName) {
-
-  // Return user plan
-  if (inputPlan !== '') {
-    return inputPlan;
-  }
-
-  // Choose a default plan based on the add-on provider
-  switch (providerName) {
-    case 'kv':
-      return 'alpha';
-    default:
-      return 'dev';
-  }
-}
-
 async function list (params) {
   const { org: orgaIdOrName, format } = params.options;
 
@@ -74,7 +57,7 @@ async function create (params) {
   const [providerName, name] = params.args;
   const {
     link: linkedAppAlias,
-    plan: inputPlan,
+    plan: planName,
     region,
     yes: skipConfirmation,
     org: orgaIdOrName,
@@ -82,7 +65,6 @@ async function create (params) {
   } = params.options;
   const version = params.options['addon-version'];
   const addonOptions = parseAddonOptions(params.options.option);
-  const planName = getPlan(inputPlan, providerName);
 
   const ownerId = (orgaIdOrName != null)
     ? await Organisation.getId(orgaIdOrName)
@@ -118,6 +100,45 @@ async function create (params) {
 }
 
 function displayAddon (format, addon, providerName, message) {
+
+  const WIP_PROVIDERS = {
+    keycloak: {
+      statut: 'alpha',
+      postCreateInstructions: [
+        'Learn more about Keycloak on Clever Cloud: https://developers.clever-cloud.com/doc/addons/keycloak/',
+      ].join('\n'),
+    },
+    kv: {
+      status: 'alpha',
+      postCreateInstructions: [
+        colors.yellow('You can easily use Materia KV with \'redis-cli\', with such commands:'),
+        colors.blue(`source <(clever addon env ${addon.id} -F shell)`),
+        colors.blue('redis-cli -h $KV_HOST -p $KV_PORT --tls'),
+        'Learn more about Materia KV on Clever Cloud: https://developers.clever-cloud.com/doc/addons/materia-kv/',
+      ].join('\n'),
+    },
+    'addon-pulsar': {
+      status: 'beta',
+      postCreateInstructions: [
+        'Learn more about Pulsar on Clever Cloud: https://developers.clever-cloud.com/doc/addons/pulsar/',
+      ].join('\n'),
+    },
+  };
+
+  let providerNameToShow = '';
+  let statusMessage = '';
+  if (providerName in WIP_PROVIDERS) {
+
+    providerNameToShow = providerName === 'kv'
+      ? 'Materia KV'
+      : providerName;
+
+    statusMessage = `The ${providerNameToShow} provider is in ${WIP_PROVIDERS[providerName].status} testing phase`;
+    statusMessage += WIP_PROVIDERS[providerName].status === 'alpha'
+      ? '. Don\'t store sensitive or production grade data.'
+      : '';
+  }
+
   switch (format) {
 
     case 'json': {
@@ -126,8 +147,9 @@ function displayAddon (format, addon, providerName, message) {
         realId: addon.realId,
         name: addon.name,
       };
-      Logger.printJson((providerName === 'kv')
-        ? { ...jsonAddon, availability: 'alpha', warning: 'Don\'t store sensitive or production grade data' }
+
+      Logger.printJson((WIP_PROVIDERS[providerName] != null)
+        ? { ...jsonAddon, availability: WIP_PROVIDERS[providerName].status, message: statusMessage }
         : jsonAddon);
       break;
     }
@@ -140,15 +162,12 @@ function displayAddon (format, addon, providerName, message) {
         `Real ID: ${addon.realId}`,
         `Name: ${addon.name}`,
       ].join('\n'));
-      if (providerName === 'kv') {
-        const materiaMessage = [
-          '',
-          colors.yellow(`/!\\ The MateriaDB ${providerName.toUpperCase()} provider is in Alpha testing phase, don't store sensitive or production grade data`),
-          'You can easily use MateriaDB KV with \'redis-cli\', with such commands:',
-          colors.blue(`source <(clever addon env ${addon.id} -F shell)`),
-          colors.blue('redis-cli -h $KV_HOST -p $KV_PORT --tls'),
-        ].join('\n');
-        Logger.println(materiaMessage);
+
+      if (providerName in WIP_PROVIDERS) {
+
+        Logger.println();
+        Logger.println(colors.yellow(`/!\\ ${statusMessage}`));
+        Logger.println(WIP_PROVIDERS[providerName].postCreateInstructions);
       }
   }
 }
@@ -173,64 +192,123 @@ async function rename (params) {
   Logger.println(`Addon ${addon.addon_id || addon.addon_name} successfully renamed to ${newName}`);
 }
 
-async function listProviders () {
+async function listProviders (params) {
+  const { format } = params.options;
 
   const providers = await Addon.listProviders();
 
-  const formattedProviders = providers.map((provider) => {
-    return [
-      colors.bold(provider.id),
-      provider.name,
-      provider.shortDesc || '',
-    ];
-  });
-  Logger.println(formatTable(formattedProviders));
+  switch (format) {
+    case 'json': {
+      const formattedProviders = providers.map((provider) => ({
+        id: provider.id,
+        name: provider.name,
+        shortDesc: provider.shortDesc,
+        regions: provider.regions,
+        plans: provider.plans.map((plan) => ({
+          id: plan.id,
+          name: plan.name,
+          slug: plan.slug,
+        })),
+      }));
+      Logger.printJson(formattedProviders);
+      break;
+    }
+    case 'human':
+    default: {
+      const formattedProviders = providers.map((provider) => {
+        return [
+          colors.bold(provider.id),
+          provider.name,
+          provider.shortDesc || '',
+        ];
+      });
+      Logger.println(formatTable(formattedProviders));
+    }
+  }
 }
 
 async function showProvider (params) {
   const [providerName] = params.args;
+  const { format } = params.options;
 
   const provider = await Addon.getProvider(providerName);
   const providerInfos = await Addon.getProviderInfos(providerName);
-  const providerPlans = provider.plans.sort((a, b) => a.price - b.price);
-
-  Logger.println(colors.bold(provider.id));
-  Logger.println(`${provider.name}: ${provider.shortDesc}`);
-  Logger.println();
-  Logger.println(`Available regions: ${provider.regions.join(', ')}`);
-  Logger.println();
-  Logger.println('Available plans');
-
-  providerPlans.forEach((plan) => {
-    Logger.println(`Plan ${colors.bold(plan.slug)}`);
-    _(plan.features)
-      .sortBy('name')
-      .forEach(({ name, value }) => Logger.println(`  ${name}: ${value}`));
-
-    if (providerInfos != null) {
-      const planType = plan.features.find(({ name }) => name.toLowerCase() === 'type');
-      if (planType != null && planType.value.toLowerCase() === 'dedicated') {
-        const planVersions = Object.keys(providerInfos.dedicated);
-        const versions = planVersions.map((version) => {
-          if (version === providerInfos.defaultDedicatedVersion) {
-            return `${version} (default)`;
-          }
-          else {
-            return version;
-          }
-        });
-        Logger.println(`  Available versions: ${versions.join(', ')}`);
-
-        planVersions.forEach((version) => {
-          const features = providerInfos.dedicated[version].features;
-          Logger.println(`  Options for version ${version}:`);
-          features.forEach(({ name, enabled }) => {
-            Logger.println(`    ${name}: default=${enabled}`);
+
+  const formattedPlans = [...provider.plans]
+    .sort((a, b) => a.price - b.price)
+    .map((plan) => {
+      const formattedFeatures = plan.features
+        .map((feature) => ({
+          name: feature.name,
+          value: feature.value,
+        }))
+        .sort((a, b) => a.name.localeCompare(b.name));
+
+      const formattedPlan = {
+        id: plan.id,
+        name: plan.name,
+        slug: plan.slug,
+        features: formattedFeatures,
+      };
+
+      if (providerInfos != null) {
+        const planType = plan.features.find(({ name }) => name.toLowerCase() === 'type');
+        if (planType != null && planType.value.toLowerCase() === 'dedicated') {
+          const planVersions = Object.keys(providerInfos.dedicated);
+          formattedPlan.versions = planVersions.map((version) => {
+            return {
+              version,
+              isDefault: version === providerInfos.defaultDedicatedVersion,
+              options: providerInfos.dedicated[version].features.map((feature) => ({
+                name: feature.name,
+                enabledByDefault: feature.enabled,
+              })),
+            };
           });
-        });
+        }
       }
+
+      return formattedPlan;
+    });
+
+  const formattedProvider = {
+    id: provider.id,
+    name: provider.name,
+    shortDesc: provider.shortDesc,
+    regions: provider.regions,
+    plans: formattedPlans,
+  };
+
+  switch (format) {
+    case 'json': {
+      Logger.printJson(formattedProvider);
+      break;
     }
-  });
+    case 'human':
+    default: {
+      Logger.println(colors.bold(formattedProvider.id));
+      Logger.println(`${formattedProvider.name}: ${formattedProvider.shortDesc}`);
+      Logger.println();
+      Logger.println(`Available regions: ${formattedProvider.regions.join(', ')}`);
+      Logger.println();
+      Logger.println('Available plans');
+
+      formattedProvider.plans.forEach((plan) => {
+        Logger.println(`Plan ${colors.bold(plan.slug)}`);
+        plan.features.forEach(({ name, value }) => Logger.println(`  ${name}: ${value}`));
+
+        if (plan.versions != null) {
+          Logger.println(`  Available versions: ${plan.versions.map(({ version, isDefault }) => isDefault ? `${version} (default)` : version).join(', ')}`);
+          plan.versions.forEach(({ version, options }) => {
+            Logger.println(`  Options for version ${version}:`);
+            options.forEach(({ name, enabledByDefault }) => {
+              Logger.println(`    ${name}: default=${enabledByDefault}`);
+            });
+          });
+        }
+      });
+    }
+  }
 }
 
 async function env (params) {