@automattic/vip 3.6.0 → 3.7.0

package/dist/bin/vip-import-validate-files.js

@@ -1,18 +1,24 @@
 #!/usr/bin/env node
+
+/**
+ * External dependencies
+ */
 "use strict";
 
 var _chalk = _interopRequireDefault(require("chalk"));
-var _fs = _interopRequireDefault(require("fs"));
-var _path = _interopRequireDefault(require("path"));
 var _url = _interopRequireDefault(require("url"));
 var _command = _interopRequireDefault(require("../lib/cli/command"));
+var _config = require("../lib/media-import/config");
 var _tracker = require("../lib/tracker");
 var _vipImportValidateFiles = require("../lib/vip-import-validate-files");
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Internal dependencies
+ */
 (0, _command.default)({
   requiredArgs: 1,
   format: true
-}).example('vip import validate-files <file>', 'Run the import validation against the file').argv(process.argv, async arg => {
+}).example('vip import validate-files <folder_name>', 'Run the import validation against the folder of media files').argv(process.argv, async arg => {
   await (0, _tracker.trackEvent)('import_validate_files_command_execute');
   /**
    * File manipulation
@@ -56,90 +62,64 @@ function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e
   }
 
   /**
-   * Media file extension validation
-   *
-   * Ensure that prohibited media file types are not used
+   * Get Media Import configuration
    */
-
-  // Collect invalid files for error logging
-  let intermediateImagesTotal = 0;
-  const errorFileTypes = [];
-  const errorFileNames = [];
-  const intermediateImages = {};
-
-  // Iterate through each file to isolate the extension name
-  for (const file of files) {
-    // Check if file is a directory
-    // eslint-disable-next-line no-await-in-loop
-    const stats = await _fs.default.promises.stat(file);
-    const isFolder = stats.isDirectory();
-    const extension = _path.default.extname(file); // Extract the extension of the file
-    const ext = extension.substr(1); // We only want the ext name minus the period (e.g- .jpg -> jpg)
-    const extLowerCase = ext.toLowerCase(); // Change any uppercase extensions to lowercase
-
-    // Check for any invalid file extensions
-    // Returns true if ext is valid; false if invalid
-    const validExtensions = _vipImportValidateFiles.acceptedExtensions.includes(extLowerCase);
-
-    // Collect files that have no extension, have invalid extensions,
-    // or are directories for error logging
-    if (!extension || !validExtensions || isFolder) {
-      errorFileTypes.push(file);
-    }
-
-    /**
-     * Filename validation
-     *
-     * Ensure that filenames don't contain prohibited characters
-     */
-
-    // Collect files that have invalid file names for error logging
-    if ((0, _vipImportValidateFiles.isFileSanitized)(file)) {
-      errorFileNames.push(file);
-    }
-
-    /**
-     * Intermediate image validation
-     *
-     * Detect any intermediate images.
-     *
-     * Intermediate images are copies of images that are resized, so you may have multiples of the same image.
-     * You can resize an image directly on VIP so intermediate images are not necessary.
-     */
-    const original = (0, _vipImportValidateFiles.doesImageHaveExistingSource)(file);
-
-    // If an image is an intermediate image, increment the total number and
-    // populate key/value pairs of the original image and intermediate image(s)
-    if (original) {
-      intermediateImagesTotal++;
-      if (intermediateImages[original]) {
-        // Key: original image, value: intermediate image(s)
-        intermediateImages[original] = `${intermediateImages[original]}, ${file}`;
-      } else {
-        intermediateImages[original] = file;
-      }
-    }
+  const mediaImportConfig = await (0, _config.getMediaImportConfig)();
+  if (!mediaImportConfig) {
+    console.error(_chalk.default.red('✕ Error:'), 'Could not retrieve validation metadata. Please contact VIP Support.');
+    return;
   }
 
+  /**
+   * File Validation
+   * Collect all errors from file validation
+   */
+  const {
+    intermediateImagesTotal,
+    errorFileTypes,
+    errorFileNames,
+    errorFileSizes,
+    errorFileNamesCharCount,
+    intermediateImages
+  } = await (0, _vipImportValidateFiles.validateFiles)(files, mediaImportConfig);
+
   /**
    * Error logging
+   * Not sure if the changes made to the error logging better
    */
-  if (errorFileTypes.length > 0) {
-    (0, _vipImportValidateFiles.logErrorsForInvalidFileTypes)(errorFileTypes);
-  }
-  if (errorFileNames.length > 0) {
-    (0, _vipImportValidateFiles.logErrorsForInvalidFilenames)(errorFileNames);
-  }
-  if (Object.keys(intermediateImages).length > 0) {
-    (0, _vipImportValidateFiles.logErrorsForIntermediateImages)(intermediateImages);
-  }
+  (0, _vipImportValidateFiles.logErrors)({
+    errorType: _vipImportValidateFiles.ValidateFilesErrors.INVALID_TYPES,
+    invalidFiles: errorFileTypes,
+    limit: Object.keys(mediaImportConfig.allowedFileTypes)
+  });
+  (0, _vipImportValidateFiles.logErrors)({
+    errorType: _vipImportValidateFiles.ValidateFilesErrors.INVALID_SIZES,
+    invalidFiles: errorFileSizes,
+    limit: mediaImportConfig.fileSizeLimitInBytes
+  });
+  (0, _vipImportValidateFiles.logErrors)({
+    errorType: _vipImportValidateFiles.ValidateFilesErrors.INVALID_NAME_CHARACTER_COUNTS,
+    invalidFiles: errorFileNamesCharCount,
+    limit: mediaImportConfig.fileNameCharCount
+  });
+  (0, _vipImportValidateFiles.logErrors)({
+    errorType: _vipImportValidateFiles.ValidateFilesErrors.INVALID_NAMES,
+    invalidFiles: errorFileNames
+  });
+  (0, _vipImportValidateFiles.logErrors)({
+    errorType: _vipImportValidateFiles.ValidateFilesErrors.INTERMEDIATE_IMAGES,
+    invalidFiles: Object.keys(intermediateImages),
+    invalidFilesObj: intermediateImages
+  });
 
   // Log a summary of all errors
   (0, _vipImportValidateFiles.summaryLogs)({
     folderErrorsLength: folderValidation.length,
     intImagesErrorsLength: intermediateImagesTotal,
     fileTypeErrorsLength: errorFileTypes.length,
+    fileErrorFileSizesLength: errorFileSizes.length,
     filenameErrorsLength: errorFileNames.length,
+    fileNameCharCountErrorsLength: errorFileNamesCharCount.length,
     totalFiles: files.length,
     totalFolders: nestedDirectories.length
   });

package/dist/lib/media-import/config.js

@@ -0,0 +1,25 @@
+"use strict";
+
+exports.__esModule = true;
+exports.getMediaImportConfig = getMediaImportConfig;
+var _graphqlTag = _interopRequireDefault(require("graphql-tag"));
+var _api = _interopRequireDefault(require("../api"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const IMPORT_MEDIA_CONFIG_QUERY = (0, _graphqlTag.default)`
+	{
+		mediaImportConfig {
+			fileNameCharCount
+			fileSizeLimitInBytes
+			allowedFileTypes
+		}
+	}
+`;
+async function getMediaImportConfig() {
+  const api = (0, _api.default)();
+  const response = await api.query({
+    query: IMPORT_MEDIA_CONFIG_QUERY,
+    variables: {},
+    fetchPolicy: 'network-only'
+  });
+  return response?.data?.mediaImportConfig;
+}

package/dist/lib/vip-import-validate-files.js

@@ -1,13 +1,118 @@
 "use strict";
 
 exports.__esModule = true;
-exports.summaryLogs = exports.logErrorsForInvalidFilenames = exports.logErrorsForInvalidFileTypes = exports.logErrorsForIntermediateImages = exports.isFileSanitized = exports.folderStructureValidation = exports.findNestedDirectories = exports.doesImageHaveExistingSource = exports.acceptedExtensions = void 0;
+exports.summaryLogs = exports.logErrors = exports.isFileSanitized = exports.folderStructureValidation = exports.findNestedDirectories = exports.doesImageHaveExistingSource = exports.ValidateFilesErrors = void 0;
+exports.validateFiles = validateFiles;
 var _chalk = _interopRequireDefault(require("chalk"));
 var _fs = _interopRequireDefault(require("fs"));
 var _path = _interopRequireDefault(require("path"));
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
-// Accepted media file extensions
-const acceptedExtensions = exports.acceptedExtensions = ['jpg', 'jpeg', 'jpe', 'gif', 'png', 'bmp', 'svg', 'tiff', 'tif', 'ico', 'asf', 'asx', 'wmv', 'wmx', 'wm', 'avi', 'divx', 'mov', 'qt', 'mpeg', 'mpg', 'mpe', 'mp4', 'm4v', 'ogv', 'webm', 'mkv', '3gp', '3gpp', '3g2', '3gp2', 'txt', 'asc', 'c', 'cc', 'h', 'srt', 'csv', 'tsv', 'ics', 'rtx', 'css', 'vtt', 'dfxp', 'mp3', 'm4a', 'm4b', 'ra', 'ram', 'wav', 'ogg', 'oga', 'mid', 'midi', 'wma', 'wax', 'mka', 'rtf', 'js', 'pdf', 'class', 'psd', 'xcf', 'doc', 'pot', 'pps', 'ppt', 'wri', 'xla', 'xls', 'xlt', 'xlw', 'mdb', 'mpp', 'docx', 'docm', 'dotx', 'dotm', 'xlsx', 'xlsm', 'xlsb', 'xltx', 'xltm', 'xlam', 'pptx', 'pptm', 'ppsx', 'ppsm', 'potx', 'potm', 'ppam', 'sldx', 'sldm', 'onetoc', ' onetoc2', 'onetmp', 'onepkg', 'oxps', 'xps', 'odt', 'odp', 'ods', 'odg', 'odc', 'odb', 'odf', 'webp', 'wp', 'wpd', 'key', 'numbers', 'pages'];
+/**
+ * External dependencies
+ */
+/**
+ * Internal dependencies
+ */
+let ValidateFilesErrors = exports.ValidateFilesErrors = /*#__PURE__*/function (ValidateFilesErrors) {
+  ValidateFilesErrors["INVALID_TYPES"] = "invalid_types";
+  ValidateFilesErrors["INTERMEDIATE_IMAGES"] = "intermediate_images";
+  ValidateFilesErrors["INVALID_SIZES"] = "invalid_sizes";
+  ValidateFilesErrors["INVALID_NAMES"] = "invalid_names";
+  ValidateFilesErrors["INVALID_NAME_CHARACTER_COUNTS"] = "invalid_name_character_counts";
+  return ValidateFilesErrors;
+}({});
+/**
+ * File info validation
+ *
+ * Validate the file info for media files
+ */
+async function validateFiles(files, mediaImportConfig) {
+  const validationResult = {
+    intermediateImagesTotal: 0,
+    errorFileTypes: [],
+    errorFileNames: [],
+    errorFileSizes: [],
+    errorFileNamesCharCount: [],
+    intermediateImages: {}
+  };
+  const fileValidationPromises = files.map(async file => {
+    const isFolder = await isDirectory(file);
+    const fileExtType = getFileExtType(file,
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    mediaImportConfig.allowedFileTypes);
+    if (isInvalidFile(fileExtType, isFolder)) {
+      validationResult.errorFileTypes.push(file);
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    if (!isFileSizeValid(file, mediaImportConfig.fileSizeLimitInBytes)) {
+      validationResult.errorFileSizes.push(file);
+    }
+    if (isFileSanitized(file)) {
+      validationResult.errorFileNames.push(file);
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    if (!isFileNameCharCountValid(file, mediaImportConfig?.fileNameCharCount)) {
+      validationResult.errorFileNamesCharCount.push(file);
+    }
+    const original = doesImageHaveExistingSource(file);
+    if (original) {
+      validationResult.intermediateImagesTotal++;
+      if (validationResult.intermediateImages[original]) {
+        validationResult.intermediateImages[original] += `, ${file}`;
+      } else {
+        validationResult.intermediateImages[original] = file;
+      }
+    }
+  });
+  await Promise.all(fileValidationPromises);
+  return validationResult;
+}
+const isDirectory = async file => {
+  const stats = await _fs.default.promises.stat(file);
+  return stats.isDirectory();
+};
+const getFileExtType = (file, allowedFileTypes) => {
+  if (!allowedFileTypes) return {
+    ext: null,
+    type: null
+  };
+  return getExtAndType(file, allowedFileTypes);
+};
+const isInvalidFile = (fileExtType, isFolder) => {
+  return !fileExtType.type || !fileExtType.ext || isFolder;
+};
+const getExtAndType = (filePath, allowedFileTypes) => {
+  const extType = {
+    ext: null,
+    type: null
+  };
+  for (const [key, value] of Object.entries(allowedFileTypes)) {
+    // Create a regular expression to match the file extension
+    // eslint-disable-next-line security/detect-non-literal-regexp
+    const regex = new RegExp(`(?:\\.)(${key})$`, 'i');
+    const matches = regex.exec(filePath);
+    if (matches) {
+      extType.type = value;
+      extType.ext = matches[1];
+      break;
+    }
+  }
+  return extType;
+};
+const isFileSizeValid = (filePathOnDisk, fileSizeLimitInBytes) => {
+  const fileStat = _fs.default.statSync(filePathOnDisk);
+  return fileSizeLimitInBytes >= fileStat.size;
+};
+const isFileNameCharCountValid = (file, fileNameCharCount) => {
+  const filename = _path.default.basename(file);
+  return filename.length <= fileNameCharCount;
+};
+
+/**
+ * End file info validation
+ */
 
 /**
  * Character validation global variables
@@ -38,8 +143,8 @@ const recommendedFileStructure = () => {
 };
 
 // Recommend accepted file types
-const recommendAcceptableFileTypes = () => {
-  console.log('Accepted file types: \n\n' + _chalk.default.magenta(`${acceptedExtensions.join(', ')}`));
+const recommendAcceptableFileTypes = allowedFileTypesString => {
+  console.log('Accepted file types: \n\n' + _chalk.default.magenta(`${allowedFileTypesString}`));
   console.log();
 };
 
@@ -392,8 +497,8 @@ const isFileSanitized = file => {
   sanitizedFile = sanitizedFile.replace(regexSpaces, ' ');
 
   // Check if the filename has been sanitized
-  const checkFile = sanitizedFile !== filename;
-  return checkFile;
+
+  return sanitizedFile !== filename;
 };
 
 /**
@@ -449,45 +554,50 @@ const doesImageHaveExistingSource = file => {
  *
  * Log errors for invalid folders or files
  */
-
-// Log errors for files with invalid file extensions and recommend accepted file types
 exports.doesImageHaveExistingSource = doesImageHaveExistingSource;
-const logErrorsForInvalidFileTypes = invalidFiles => {
-  invalidFiles.forEach(file => {
-    console.error(_chalk.default.red('✕'), 'File extensions: Invalid file type for file: ', _chalk.default.cyan(`${file}`));
-  });
-  console.log();
-  recommendAcceptableFileTypes();
-  console.log('------------------------------------------------------------');
-  console.log();
-};
-
-// Log errors for files with invalid filenames and show a list of accepted/prohibited chars
-exports.logErrorsForInvalidFileTypes = logErrorsForInvalidFileTypes;
-const logErrorsForInvalidFilenames = invalidFiles => {
+const logErrors = ({
+  errorType,
+  invalidFiles,
+  limit
+}) => {
+  if (invalidFiles.length === 0) {
+    return;
+  }
   invalidFiles.forEach(file => {
-    console.error(_chalk.default.red('✕'), 'Character validation: Invalid filename for file: ', _chalk.default.cyan(`${file}`));
+    switch (errorType) {
+      case ValidateFilesErrors.INVALID_TYPES:
+        console.error(_chalk.default.red('✕'), 'File extensions: Invalid file type for file: ', _chalk.default.cyan(`${file}`));
+        console.log();
+        recommendAcceptableFileTypes(limit);
+        break;
+      case ValidateFilesErrors.INTERMEDIATE_IMAGES:
+        console.error(_chalk.default.red('✕'), 'Intermediate images: Duplicate files found:\n' + 'Original file: ' + _chalk.default.blue(`${file}\n`) + 'Intermediate images: ' + _chalk.default.cyan(`${limit[file]}\n`));
+        break;
+      case ValidateFilesErrors.INVALID_SIZES:
+        console.error(_chalk.default.red('✕'), `File size cannot be more than ${limit / 1024 / 1024 / 1024} GB`, _chalk.default.cyan(`${file}`));
+        console.log();
+        break;
+      case ValidateFilesErrors.INVALID_NAME_CHARACTER_COUNTS:
+        console.error(_chalk.default.red('✕'), `File name cannot have more than ${limit} characters`, _chalk.default.cyan(`${file}`));
+        break;
+      case ValidateFilesErrors.INVALID_NAMES:
+        console.error(_chalk.default.red('✕'), 'Character validation: Invalid filename for file: ', _chalk.default.cyan(`${file}`));
+        recommendAcceptableFileNames();
+        break;
+      default:
+        console.error(_chalk.default.red('✕'), 'Unknown error type:', errorType);
+    }
   });
   console.log();
-  recommendAcceptableFileNames();
-  console.log('------------------------------------------------------------');
-  console.log();
-};
-
-// Log errors for intermediate image file duplicates
-exports.logErrorsForInvalidFilenames = logErrorsForInvalidFilenames;
-const logErrorsForIntermediateImages = obj => {
-  for (const original in obj) {
-    console.error(_chalk.default.red('✕'), 'Intermediate images: Duplicate files found:\n' + 'Original file: ' + _chalk.default.blue(`${original}\n`) + 'Intermediate images: ' + _chalk.default.cyan(`${obj[original]}\n`));
-  }
-  console.log('------------------------------------------------------------');
 };
-exports.logErrorsForIntermediateImages = logErrorsForIntermediateImages;
+exports.logErrors = logErrors;
 const summaryLogs = ({
   folderErrorsLength,
   intImagesErrorsLength,
   fileTypeErrorsLength,
+  fileErrorFileSizesLength,
   filenameErrorsLength,
+  fileNameCharCountErrorsLength,
   totalFiles,
   totalFolders
 }) => {
@@ -507,11 +617,21 @@ const summaryLogs = ({
   } else {
     messages.push(_chalk.default.white.bgGreen('    PASS     ') + _chalk.default.green(` ${fileTypeErrorsLength} invalid file extensions`) + `, ${totalFiles} files total`);
   }
+  if (fileErrorFileSizesLength > 0) {
+    messages.push(_chalk.default.white.bgRed('   ERROR     ') + _chalk.default.red(` ${fileTypeErrorsLength} invalid file sizes`) + `, ${totalFiles} files total`);
+  } else {
+    messages.push(_chalk.default.white.bgGreen('    PASS     ') + _chalk.default.green(` ${fileTypeErrorsLength} invalid file sizes`) + `, ${totalFiles} files total`);
+  }
   if (filenameErrorsLength) {
     messages.push(_chalk.default.white.bgRed('   ERROR     ') + _chalk.default.red(` ${filenameErrorsLength} invalid filenames`) + `, ${totalFiles} files total`);
   } else {
     messages.push(_chalk.default.bgGreen('    PASS     ') + _chalk.default.green(` ${filenameErrorsLength} invalid filenames`) + `, ${totalFiles} files total`);
   }
+  if (fileNameCharCountErrorsLength) {
+    messages.push(_chalk.default.white.bgRed('   ERROR     ') + _chalk.default.red(` ${filenameErrorsLength} file names reached the maximum character count limit `) + `, ${totalFiles} files total`);
+  } else {
+    messages.push(_chalk.default.bgGreen('    PASS     ') + _chalk.default.green(` ${filenameErrorsLength} file names reached the maximum character count limit`) + `, ${totalFiles} files total`);
+  }
   console.log(`\n${messages.join('\n')}\n`);
 };
 exports.summaryLogs = summaryLogs;