@strapi/strapi 4.6.0-beta.1 → 4.6.0

package/lib/commands/transfer/export.js

@@ -1,25 +1,27 @@
 'use strict';
 
 const {
-  createLocalFileDestinationProvider,
-  createLocalStrapiSourceProvider,
-  createTransferEngine,
-  // TODO: we need to solve this issue with typescript modules
-  // eslint-disable-next-line import/no-unresolved, node/no-missing-require
-} = require('@strapi/data-transfer');
+  providers: { createLocalFileDestinationProvider },
+} = require('@strapi/data-transfer/lib/file');
+const {
+  providers: { createLocalStrapiSourceProvider },
+} = require('@strapi/data-transfer/lib/strapi');
+const { createTransferEngine } = require('@strapi/data-transfer/lib/engine');
 const { isObject, isString, isFinite, toNumber } = require('lodash/fp');
 const fs = require('fs-extra');
 const chalk = require('chalk');
 
+const { TransferEngineTransferError } = require('@strapi/data-transfer/lib/engine/errors');
 const {
   getDefaultExportName,
   buildTransferTable,
   DEFAULT_IGNORED_CONTENT_TYPES,
   createStrapiInstance,
+  formatDiagnostic,
 } = require('./utils');
 
 /**
- * @typedef ImportCommandOptions Options given to the CLI import command
+ * @typedef ExportCommandOptions Options given to the CLI import command
  *
  * @property {string} [file] The file path to import
  * @property {boolean} [encrypt] Used to encrypt the final archive
@@ -32,11 +34,11 @@ const logger = console;
 const BYTES_IN_MB = 1024 * 1024;
 
 /**
- * Import command.
+ * Export command.
  *
- * It transfers data from a local file to a local strapi instance
+ * It transfers data from a local Strapi instance to a file
  *
- * @param {ImportCommandOptions} opts
+ * @param {ExportCommandOptions} opts
  */
 module.exports = async (opts) => {
   // Validate inputs from Commander
@@ -53,6 +55,8 @@ module.exports = async (opts) => {
   const engine = createTransferEngine(source, destination, {
     versionStrategy: 'ignore', // for an export to file, versionStrategy will always be skipped
     schemaStrategy: 'ignore', // for an export to file, schemaStrategy will always be skipped
+    exclude: opts.exclude,
+    only: opts.only,
     transforms: {
       links: [
         {
@@ -74,32 +78,25 @@ module.exports = async (opts) => {
     },
   });
 
-  try {
-    logger.log(`Starting export...`);
+  engine.diagnostics.onDiagnostic(formatDiagnostic('export'));
 
-    const progress = engine.progress.stream;
+  const progress = engine.progress.stream;
 
-    const telemetryPayload = (/* payload */) => {
-      return {
-        eventProperties: {
-          source: engine.sourceProvider.name,
-          destination: engine.destinationProvider.name,
-        },
-      };
+  const getTelemetryPayload = (/* payload */) => {
+    return {
+      eventProperties: {
+        source: engine.sourceProvider.name,
+        destination: engine.destinationProvider.name,
+      },
     };
+  };
 
-    progress.on('transfer::start', (payload) => {
-      strapi.telemetry.send('didDEITSProcessStart', telemetryPayload(payload));
-    });
-
-    progress.on('transfer::finish', (payload) => {
-      strapi.telemetry.send('didDEITSProcessFinish', telemetryPayload(payload));
-    });
-
-    progress.on('transfer::error', (payload) => {
-      strapi.telemetry.send('didDEITSProcessFail', telemetryPayload(payload));
-    });
+  progress.on('transfer::start', async () => {
+    logger.log(`Starting export...`);
+    await strapi.telemetry.send('didDEITSProcessStart', getTelemetryPayload());
+  });
 
+  try {
     const results = await engine.transfer();
     const outFile = results.destination.file.path;
 
@@ -108,16 +105,20 @@ module.exports = async (opts) => {
 
     const outFileExists = await fs.pathExists(outFile);
     if (!outFileExists) {
-      throw new Error(`Export file not created "${outFile}"`);
+      throw new TransferEngineTransferError(`Export file not created "${outFile}"`);
     }
 
     logger.log(`${chalk.bold('Export process has been completed successfully!')}`);
     logger.log(`Export archive is in ${chalk.green(outFile)}`);
-    process.exit(0);
-  } catch (e) {
-    logger.error('Export process failed unexpectedly:', e.toString());
+  } catch {
+    await strapi.telemetry.send('didDEITSProcessFail', getTelemetryPayload());
+    logger.error('Export process failed.');
     process.exit(1);
   }
+
+  // Note: Telemetry can't be sent in a finish event, because it runs async after this block but we can't await it, so if process.exit is used it won't send
+  await strapi.telemetry.send('didDEITSProcessFinish', getTelemetryPayload());
+  process.exit(0);
 };
 
 /**
@@ -134,7 +135,7 @@ const createSourceProvider = (strapi) => {
 /**
  * It creates a local file destination provider based on the given options
  *
- * @param {ImportCommandOptions} opts
+ * @param {ExportCommandOptions} opts
  */
 const createDestinationProvider = (opts) => {
   const { file, compress, encrypt, key, maxSizeJsonl } = opts;

package/lib/commands/transfer/import.js

@@ -1,20 +1,25 @@
 'use strict';
 
 const {
-  createLocalFileSourceProvider,
-  createLocalStrapiDestinationProvider,
+  providers: { createLocalFileSourceProvider },
+} = require('@strapi/data-transfer/lib/file');
+const {
+  providers: { createLocalStrapiDestinationProvider, DEFAULT_CONFLICT_STRATEGY },
+} = require('@strapi/data-transfer/lib/strapi');
+const {
   createTransferEngine,
   DEFAULT_VERSION_STRATEGY,
   DEFAULT_SCHEMA_STRATEGY,
-  DEFAULT_CONFLICT_STRATEGY,
-  // TODO: we need to solve this issue with typescript modules
-  // eslint-disable-next-line import/no-unresolved, node/no-missing-require
-} = require('@strapi/data-transfer');
+} = require('@strapi/data-transfer/lib/engine');
+
 const { isObject } = require('lodash/fp');
-const path = require('path');
 
-const strapi = require('../../index');
-const { buildTransferTable, DEFAULT_IGNORED_CONTENT_TYPES } = require('./utils');
+const {
+  buildTransferTable,
+  DEFAULT_IGNORED_CONTENT_TYPES,
+  createStrapiInstance,
+  formatDiagnostic,
+} = require('./utils');
 
 /**
  * @typedef {import('@strapi/data-transfer').ILocalFileSourceProviderOptions} ILocalFileSourceProviderOptions
@@ -39,12 +44,13 @@ module.exports = async (opts) => {
   /**
    * To local Strapi instance
    */
-  const strapiInstance = await strapi(await strapi.compile()).load();
+  const strapiInstance = await createStrapiInstance();
 
   const destinationOptions = {
     async getStrapi() {
       return strapiInstance;
     },
+    autoDestroy: false,
     strategy: opts.conflictStrategy || DEFAULT_CONFLICT_STRATEGY,
     restore: {
       entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES },
@@ -59,6 +65,7 @@ module.exports = async (opts) => {
     versionStrategy: opts.versionStrategy || DEFAULT_VERSION_STRATEGY,
     schemaStrategy: opts.schemaStrategy || DEFAULT_SCHEMA_STRATEGY,
     exclude: opts.exclude,
+    only: opts.only,
     rules: {
       links: [
         {
@@ -77,44 +84,43 @@ module.exports = async (opts) => {
       ],
     },
   };
+
   const engine = createTransferEngine(source, destination, engineOptions);
 
-  try {
-    logger.info('Starting import...');
+  engine.diagnostics.onDiagnostic(formatDiagnostic('import'));
 
-    const progress = engine.progress.stream;
-    const telemetryPayload = (/* payload */) => {
-      return {
-        eventProperties: {
-          source: engine.sourceProvider.name,
-          destination: engine.destinationProvider.name,
-        },
-      };
+  const progress = engine.progress.stream;
+  const getTelemetryPayload = () => {
+    return {
+      eventProperties: {
+        source: engine.sourceProvider.name,
+        destination: engine.destinationProvider.name,
+      },
     };
+  };
 
-    progress.on('transfer::start', (payload) => {
-      strapiInstance.telemetry.send('didDEITSProcessStart', telemetryPayload(payload));
-    });
-
-    progress.on('transfer::finish', (payload) => {
-      strapiInstance.telemetry.send('didDEITSProcessFinish', telemetryPayload(payload));
-    });
-
-    progress.on('transfer::error', (payload) => {
-      strapiInstance.telemetry.send('didDEITSProcessFail', telemetryPayload(payload));
-    });
+  progress.on('transfer::start', async () => {
+    logger.info('Starting import...');
+    await strapiInstance.telemetry.send('didDEITSProcessStart', getTelemetryPayload());
+  });
 
+  try {
     const results = await engine.transfer();
     const table = buildTransferTable(results.engine);
     logger.info(table.toString());
 
     logger.info('Import process has been completed successfully!');
-    process.exit(0);
   } catch (e) {
-    logger.error('Import process failed unexpectedly:');
-    logger.error(e);
+    await strapiInstance.telemetry.send('didDEITSProcessFail', getTelemetryPayload());
+    logger.error('Import process failed.');
     process.exit(1);
   }
+
+  // Note: Telemetry can't be sent in a finish event, because it runs async after this block but we can't await it, so if process.exit is used it won't send
+  await strapiInstance.telemetry.send('didDEITSProcessFinish', getTelemetryPayload());
+  await strapiInstance.destroy();
+
+  process.exit(0);
 };
 
 /**
@@ -130,23 +136,9 @@ const getLocalFileSourceOptions = (opts) => {
    */
   const options = {
     file: { path: opts.file },
-    compression: { enabled: false },
-    encryption: { enabled: false },
+    compression: { enabled: !!opts.decompress },
+    encryption: { enabled: !!opts.decrypt, key: opts.key },
   };
 
-  const { extname, parse } = path;
-
-  let file = options.file.path;
-
-  if (extname(file) === '.enc') {
-    file = parse(file).name;
-    options.encryption = { enabled: true, key: opts.key };
-  }
-
-  if (extname(file) === '.gz') {
-    file = parse(file).name;
-    options.compression = { enabled: true };
-  }
-
   return options;
 };

package/lib/commands/transfer/transfer.js

@@ -0,0 +1,129 @@
+'use strict';
+
+const { createTransferEngine } = require('@strapi/data-transfer/lib/engine');
+const {
+  providers: {
+    createRemoteStrapiDestinationProvider,
+    createLocalStrapiSourceProvider,
+    createLocalStrapiDestinationProvider,
+  },
+} = require('@strapi/data-transfer/lib/strapi');
+const { isObject } = require('lodash/fp');
+const chalk = require('chalk');
+
+const {
+  buildTransferTable,
+  createStrapiInstance,
+  DEFAULT_IGNORED_CONTENT_TYPES,
+  formatDiagnostic,
+} = require('./utils');
+
+const logger = console;
+
+/**
+ * @typedef TransferCommandOptions Options given to the CLI transfer command
+ *
+ * @property {URL|undefined} [to] The url of a remote Strapi to use as remote destination
+ * @property {URL|undefined} [from] The url of a remote Strapi to use as remote source
+ */
+
+/**
+ * Transfer command.
+ *
+ * It transfers data from a local file to a local strapi instance
+ *
+ * @param {TransferCommandOptions} opts
+ */
+module.exports = async (opts) => {
+  // Validate inputs from Commander
+  if (!isObject(opts)) {
+    logger.error('Could not parse command arguments');
+    process.exit(1);
+  }
+
+  const strapi = await createStrapiInstance();
+
+  let source;
+  let destination;
+
+  if (!opts.from && !opts.to) {
+    logger.error('At least one source (from) or destination (to) option must be provided');
+    process.exit(1);
+  }
+
+  // if no URL provided, use local Strapi
+  if (!opts.from) {
+    source = createLocalStrapiSourceProvider({
+      getStrapi: () => strapi,
+    });
+  }
+  // if URL provided, set up a remote source provider
+  else {
+    logger.error(`Remote Strapi source provider not yet implemented`);
+    process.exit(1);
+  }
+
+  // if no URL provided, use local Strapi
+  if (!opts.to) {
+    destination = createLocalStrapiDestinationProvider({
+      getStrapi: () => strapi,
+    });
+  }
+  // if URL provided, set up a remote destination provider
+  else {
+    destination = createRemoteStrapiDestinationProvider({
+      url: opts.to,
+      auth: false,
+      strategy: 'restore',
+      restore: {
+        entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES },
+      },
+    });
+  }
+
+  if (!source || !destination) {
+    logger.error('Could not create providers');
+    process.exit(1);
+  }
+
+  const engine = createTransferEngine(source, destination, {
+    versionStrategy: 'strict',
+    schemaStrategy: 'strict',
+    transforms: {
+      links: [
+        {
+          filter(link) {
+            return (
+              !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.left.type) &&
+              !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.right.type)
+            );
+          },
+        },
+      ],
+      entities: [
+        {
+          filter(entity) {
+            return !DEFAULT_IGNORED_CONTENT_TYPES.includes(entity.type);
+          },
+        },
+      ],
+    },
+  });
+
+  engine.diagnostics.onDiagnostic(formatDiagnostic('transfer'));
+
+  try {
+    logger.log(`Starting transfer...`);
+
+    const results = await engine.transfer();
+
+    const table = buildTransferTable(results.engine);
+    logger.log(table.toString());
+
+    logger.log(`${chalk.bold('Transfer process has been completed successfully!')}`);
+    process.exit(0);
+  } catch (e) {
+    logger.error('Transfer process failed.');
+    process.exit(1);
+  }
+};

package/lib/commands/transfer/utils.js

@@ -2,8 +2,16 @@
 
 const chalk = require('chalk');
 const Table = require('cli-table3');
-const { readableBytes } = require('../utils');
+const { Option } = require('commander');
+const { TransferGroupPresets } = require('@strapi/data-transfer/lib/engine');
+
+const {
+  configs: { createOutputFileConfiguration },
+  createLogger,
+} = require('@strapi/logger');
+const { readableBytes, exitWith } = require('../utils/helpers');
 const strapi = require('../../index');
+const { getParseListWithChoices } = require('../utils/commander');
 
 const pad = (n) => {
   return (n < 10 ? '0' : '') + String(n);
@@ -74,21 +82,100 @@ const DEFAULT_IGNORED_CONTENT_TYPES = [
   'admin::role',
   'admin::api-token',
   'admin::api-token-permission',
+  'admin::audit-log',
 ];
 
 const createStrapiInstance = async (logLevel = 'error') => {
-  const appContext = await strapi.compile();
-  const app = strapi(appContext);
+  try {
+    const appContext = await strapi.compile();
+    const app = strapi(appContext);
+
+    app.log.level = logLevel;
+    return await app.load();
+  } catch (err) {
+    if (err.code === 'ECONNREFUSED') {
+      throw new Error('Process failed. Check the database connection with your Strapi project.');
+    }
+    throw err;
+  }
+};
+
+const transferDataTypes = Object.keys(TransferGroupPresets);
+
+const excludeOption = new Option(
+  '--exclude <comma-separated data types>',
+  `Exclude data using comma-separated types. Available types: ${transferDataTypes.join(',')}`
+).argParser(getParseListWithChoices(transferDataTypes, 'Invalid options for "exclude"'));
+
+const onlyOption = new Option(
+  '--only <command-separated data types>',
+  `Include only these types of data (plus schemas). Available types: ${transferDataTypes.join(',')}`
+).argParser(getParseListWithChoices(transferDataTypes, 'Invalid options for "only"'));
 
-  app.log.level = logLevel;
+const validateExcludeOnly = (command) => {
+  const { exclude, only } = command.opts();
+  if (!only || !exclude) {
+    return;
+  }
 
-  return app.load();
+  const choicesInBoth = only.filter((n) => {
+    return exclude.indexOf(n) !== -1;
+  });
+  if (choicesInBoth.length > 0) {
+    exitWith(
+      1,
+      `Data types may not be used in both "exclude" and "only" in the same command. Found in both: ${choicesInBoth.join(
+        ','
+      )}`
+    );
+  }
+};
+
+const errorColors = {
+  fatal: chalk.red,
+  error: chalk.red,
+  silly: chalk.yellow,
 };
 
+const formatDiagnostic =
+  (operation) =>
+  ({ details, kind }) => {
+    const logger = createLogger(
+      createOutputFileConfiguration(`${operation}_error_log_${Date.now()}.log`)
+    );
+    try {
+      if (kind === 'error') {
+        const { message, severity = 'fatal' } = details;
+
+        const colorizeError = errorColors[severity];
+        const errorMessage = colorizeError(`[${severity.toUpperCase()}] ${message}`);
+
+        logger.error(errorMessage);
+      }
+      if (kind === 'info') {
+        const { message, params } = details;
+
+        const msg = `${message}\n${params ? JSON.stringify(params, null, 2) : ''}`;
+
+        logger.info(msg);
+      }
+      if (kind === 'warning') {
+        const { origin, message } = details;
+
+        logger.warn(`(${origin ?? 'transfer'}) ${message}`);
+      }
+    } catch (err) {
+      logger.error(err);
+    }
+  };
+
 module.exports = {
   buildTransferTable,
   getDefaultExportName,
-  yyyymmddHHMMSS,
   DEFAULT_IGNORED_CONTENT_TYPES,
   createStrapiInstance,
+  excludeOption,
+  onlyOption,
+  validateExcludeOnly,
+  formatDiagnostic,
 };

package/lib/commands/utils/commander.js

@@ -1,12 +1,59 @@
 'use strict';
 
+/**
+ * This file includes hooks to use for commander.hook and argParsers for commander.argParser
+ */
+
 const inquirer = require('inquirer');
+const { InvalidOptionArgumentError, Option } = require('commander');
+const { bold, green, cyan } = require('chalk');
+const { exitWith } = require('./helpers');
+
+/**
+ * argParser: Parse a comma-delimited string as an array
+ */
+const parseList = (value) => {
+  let list;
+  try {
+    list = value.split(',').map((item) => item.trim()); // trim shouldn't be necessary but might help catch unexpected whitespace characters
+  } catch (e) {
+    exitWith(1, `Unrecognized input: ${value}`);
+  }
+  return list;
+};
+
+/**
+ * Returns an argParser that returns a list
+ */
+const getParseListWithChoices = (choices, errorMessage = 'Invalid options:') => {
+  return (value) => {
+    const list = parseList(value);
+    const invalid = list.filter((item) => {
+      return !choices.includes(item);
+    });
+
+    if (invalid.length > 0) {
+      exitWith(1, `${errorMessage}: ${invalid.join(',')}`);
+    }
+
+    return list;
+  };
+};
 
 /**
- * argsParser: Parse a comma-delimited string as an array
+ * argParser: Parse a string as a URL object
  */
-const parseInputList = (value) => {
-  return value.split(',');
+const parseURL = (value) => {
+  try {
+    const url = new URL(value);
+    if (!url.host) {
+      throw new InvalidOptionArgumentError(`Could not parse url ${value}`);
+    }
+
+    return url;
+  } catch (e) {
+    throw new InvalidOptionArgumentError(`Could not parse url ${value}`);
+  }
 };
 
 /**
@@ -16,8 +63,7 @@ const promptEncryptionKey = async (thisCommand) => {
   const opts = thisCommand.opts();
 
   if (!opts.encrypt && opts.key) {
-    console.error('Key may not be present unless encryption is used');
-    process.exit(1);
+    return exitWith(1, 'Key may not be present unless encryption is used');
   }
 
   // if encrypt==true but we have no key, prompt for it
@@ -37,21 +83,30 @@ const promptEncryptionKey = async (thisCommand) => {
       ]);
       opts.key = answers.key;
     } catch (e) {
-      console.error('Failed to get encryption key');
-      process.exit(1);
+      return exitWith(1, 'Failed to get encryption key');
     }
     if (!opts.key) {
-      console.error('Failed to get encryption key');
-      process.exit(1);
+      return exitWith(1, 'Failed to get encryption key');
     }
   }
 };
 
 /**
- * hook: require a confirmation message to be accepted
+ * hook: require a confirmation message to be accepted unless forceOption (-f,--force) is used
+ *
+ * @param {string} message The message to confirm with user
+ * @param {object} options Additional options
  */
 const confirmMessage = (message) => {
-  return async () => {
+  return async (command) => {
+    // if we have a force option, assume yes
+    const opts = command.opts();
+    if (opts?.force === true) {
+      // attempt to mimic the inquirer prompt exactly
+      console.log(`${green('?')} ${bold(message)} ${cyan('Yes')}`);
+      return;
+    }
+
     const answers = await inquirer.prompt([
       {
         type: 'confirm',
@@ -61,13 +116,21 @@ const confirmMessage = (message) => {
       },
     ]);
     if (!answers.confirm) {
-      process.exit(0);
+      exitWith(0);
     }
   };
 };
 
+const forceOption = new Option(
+  '--force',
+  `Automatically answer "yes" to all prompts, including potentially destructive requests, and run non-interactively.`
+);
+
 module.exports = {
-  parseInputList,
+  getParseListWithChoices,
+  parseList,
+  parseURL,
   promptEncryptionKey,
   confirmMessage,
+  forceOption,
 };