@webiny/api-page-builder-import-export 0.0.0-unstable.d7f521b032 → 0.0.0-unstable.de38392959

package/import/utils.js

@@ -1,607 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.importBlock = importBlock;
-exports.importPage = importPage;
-exports.importTemplate = importTemplate;
-exports.initialStats = initialStats;
-exports.readExtractAndUploadZipFileContents = readExtractAndUploadZipFileContents;
-exports.uploadAssets = void 0;
-var _uniqid = _interopRequireDefault(require("uniqid"));
-var _dotPropImmutable = _interopRequireDefault(require("dot-prop-immutable"));
-var _fs = require("fs");
-var _fsExtra = require("fs-extra");
-var _util = require("util");
-var _stream = require("stream");
-var _nodeFetch = _interopRequireDefault(require("node-fetch"));
-var _path = _interopRequireDefault(require("path"));
-var _yauzl = _interopRequireDefault(require("yauzl"));
-var _chunk = _interopRequireDefault(require("lodash/chunk"));
-var _loadJsonFile = _interopRequireDefault(require("load-json-file"));
-var _error = _interopRequireDefault(require("@webiny/error"));
-var _downloadInstallFiles = require("@webiny/api-page-builder/graphql/crud/install/utils/downloadInstallFiles");
-var _types = require("../types");
-var _s3Stream = require("../export/s3Stream");
-const streamPipeline = (0, _util.promisify)(_stream.pipeline);
-const INSTALL_DIR = "/tmp";
-const INSTALL_EXTRACT_DIR = _path.default.join(INSTALL_DIR, "apiPageBuilderImport");
-const FILES_COUNT_IN_EACH_BATCH = 15;
-function updateImageInPageSettings(params) {
-  const {
-    settings,
-    fileIdToKeyMap,
-    srcPrefix
-  } = params;
-  let newSettings = settings;
-  const srcPrefixWithoutTrailingSlash = srcPrefix.endsWith("/") ? srcPrefix.slice(0, -1) : srcPrefix;
-  if (_dotPropImmutable.default.get(newSettings, "general.image.src")) {
-    var _settings$general, _settings$general$ima;
-    newSettings = _dotPropImmutable.default.set(newSettings, "general.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$general = settings.general) === null || _settings$general === void 0 ? void 0 : (_settings$general$ima = _settings$general.image) === null || _settings$general$ima === void 0 ? void 0 : _settings$general$ima.id) || "")}`);
-  }
-  if (_dotPropImmutable.default.get(newSettings, "social.image.src")) {
-    var _settings$social, _settings$social$imag;
-    newSettings = _dotPropImmutable.default.set(newSettings, "social.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$social = settings.social) === null || _settings$social === void 0 ? void 0 : (_settings$social$imag = _settings$social.image) === null || _settings$social$imag === void 0 ? void 0 : _settings$social$imag.id) || "")}`);
-  }
-  return newSettings;
-}
-function updateBlockPreviewImage(params) {
-  const {
-    file,
-    fileIdToKeyMap,
-    srcPrefix
-  } = params;
-  const newFile = file;
-  const srcPrefixWithoutTrailingSlash = srcPrefix.endsWith("/") ? srcPrefix.slice(0, -1) : srcPrefix;
-  newFile.src = `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(file.id || "")}`;
-  return newFile;
-}
-function updateFilesInData({
-  data,
-  fileIdToKeyMap,
-  srcPrefix
-}) {
-  // BASE CASE: Termination point
-  if (!data || typeof data !== "object") {
-    return;
-  }
-  // Recursively call function if data is array
-  if (Array.isArray(data)) {
-    for (let i = 0; i < data.length; i++) {
-      const element = data[i];
-      updateFilesInData({
-        data: element,
-        fileIdToKeyMap,
-        srcPrefix
-      });
-    }
-    return;
-  }
-  // Main logic
-  const tuple = Object.entries(data);
-  for (let i = 0; i < tuple.length; i++) {
-    const [key, value] = tuple[i];
-    if (key === "file" && value && fileIdToKeyMap.has(value.id)) {
-      value.key = fileIdToKeyMap.get(value.id);
-      value.name = fileIdToKeyMap.get(value.id);
-      value.src = `${srcPrefix}${srcPrefix.endsWith("/") ? "" : "/"}${fileIdToKeyMap.get(value.id)}`;
-    } else {
-      updateFilesInData({
-        data: value,
-        srcPrefix,
-        fileIdToKeyMap
-      });
-    }
-  }
-}
-const uploadAssets = async params => {
-  const {
-    context,
-    filesData,
-    fileUploadsData
-  } = params;
-  // Save uploaded file key against static id for later use.
-  const fileIdToKeyMap = new Map();
-  /**
-   * This function contains logic of file download from S3.
-   * Current we're not mocking zip file download from S3 in tests at the moment.
-   * So, we're manually mocking it in case of test just by returning an empty object.
-   */
-  if (process.env.NODE_ENV === "test") {
-    return {
-      fileIdToKeyMap
-    };
-  }
-
-  // Save files meta data against old key for later use.
-  const fileKeyToFileMap = new Map();
-  // Initialize maps.
-  for (let i = 0; i < filesData.length; i++) {
-    const file = filesData[i];
-    fileKeyToFileMap.set(file.key, file);
-
-    // Initialize the value
-    fileIdToKeyMap.set(file.id, file.type);
-  }
-  const fileUploadResults = await uploadFilesFromS3({
-    fileKeyToFileMap,
-    oldKeyToNewKeyMap: fileUploadsData.assets
-  });
-
-  // Create files in File Manager
-  const createFilesInput = fileUploadResults.map(uploadResult => {
-    const newKey = uploadResult.Key;
-    const file = fileKeyToFileMap.get(getOldFileKey(newKey));
-    if (!file) {
-      return null;
-    }
-
-    // Update the file map with newly uploaded file.
-    fileIdToKeyMap.set(file.id, newKey);
-    return {
-      key: newKey,
-      name: file.name,
-      size: file.size,
-      type: file.type,
-      meta: file.meta,
-      tags: file.tags
-    };
-  }).filter(Boolean);
-  const createFilesPromises = [];
-  // Gives an array of chunks (each consists of FILES_COUNT_IN_EACH_BATCH items).
-  const createFilesInputChunks = (0, _chunk.default)(createFilesInput, FILES_COUNT_IN_EACH_BATCH);
-  for (let i = 0; i < createFilesInputChunks.length; i++) {
-    const createFilesInputChunk = createFilesInputChunks[i];
-    createFilesPromises.push(
-    /*
-     * We need to break down files into chunks because
-     * `createFilesInBatch` operation has a limit on number of files it can handle at once.
-     */
-    context.fileManager.files.createFilesInBatch(createFilesInputChunk));
-  }
-  await Promise.all(createFilesPromises);
-  return {
-    fileIdToKeyMap
-  };
-};
-exports.uploadAssets = uploadAssets;
-async function importPage({
-  pageKey,
-  context,
-  fileUploadsData
-}) {
-  const log = console.log;
-
-  // Making Directory for page in which we're going to extract the page data file.
-  const PAGE_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, pageKey);
-  (0, _fsExtra.ensureDirSync)(PAGE_EXTRACT_DIR);
-  const pageDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
-  const PAGE_DATA_FILE_PATH = _path.default.join(PAGE_EXTRACT_DIR, _path.default.basename(pageDataFileKey));
-  log(`Downloading Page data file: ${pageDataFileKey} at "${PAGE_DATA_FILE_PATH}"`);
-  // Download and save page data file in disk.
-  await new Promise((resolve, reject) => {
-    _s3Stream.s3Stream.readStream(pageDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(PAGE_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
-  });
-
-  // Load the page data file from disk.
-  log(`Load file ${pageDataFileKey}`);
-  const {
-    page,
-    files
-  } = await (0, _loadJsonFile.default)(PAGE_DATA_FILE_PATH);
-
-  // Only update page data if there are files.
-  if (files && Array.isArray(files) && files.length > 0) {
-    // Upload page assets.
-    const {
-      fileIdToKeyMap
-    } = await uploadAssets({
-      context,
-      /**
-       * TODO @ts-refactor @ashutosh figure out correct types.
-       */
-      // @ts-ignore
-      filesData: files,
-      fileUploadsData
-    });
-    const settings = await context.fileManager.settings.getSettings();
-    const {
-      srcPrefix = ""
-    } = settings || {};
-    updateFilesInData({
-      data: page.content || {},
-      fileIdToKeyMap,
-      srcPrefix
-    });
-    page.settings = updateImageInPageSettings({
-      settings: page.settings || {},
-      fileIdToKeyMap,
-      srcPrefix
-    });
-  }
-  log("Removing Directory for page...");
-  await (0, _downloadInstallFiles.deleteFile)(pageKey);
-  log(`Remove page contents from S3...`);
-  await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
-  return page;
-}
-async function importBlock({
-  blockKey,
-  context,
-  fileUploadsData
-}) {
-  const log = console.log;
-
-  // Making Directory for block in which we're going to extract the block data file.
-  const BLOCK_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, blockKey);
-  (0, _fsExtra.ensureDirSync)(BLOCK_EXTRACT_DIR);
-  const blockDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
-  const BLOCK_DATA_FILE_PATH = _path.default.join(BLOCK_EXTRACT_DIR, _path.default.basename(blockDataFileKey));
-  log(`Downloading Block data file: ${blockDataFileKey} at "${BLOCK_DATA_FILE_PATH}"`);
-  // Download and save block data file in disk.
-  await new Promise((resolve, reject) => {
-    _s3Stream.s3Stream.readStream(blockDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(BLOCK_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
-  });
-
-  // Load the block data file from disk.
-  log(`Load file ${blockDataFileKey}`);
-  const {
-    block,
-    files
-  } = await (0, _loadJsonFile.default)(BLOCK_DATA_FILE_PATH);
-
-  // Only update block data if there are files.
-  if (files && Array.isArray(files) && files.length > 0) {
-    // Upload block assets.
-    const {
-      fileIdToKeyMap
-    } = await uploadAssets({
-      context,
-      filesData: files,
-      fileUploadsData
-    });
-    const settings = await context.fileManager.settings.getSettings();
-    const {
-      srcPrefix = ""
-    } = settings || {};
-    updateFilesInData({
-      data: block.content || {},
-      fileIdToKeyMap,
-      srcPrefix
-    });
-    block.preview = updateBlockPreviewImage({
-      file: block.preview || {},
-      fileIdToKeyMap,
-      srcPrefix
-    });
-  }
-  log("Removing Directory for block...");
-  await (0, _downloadInstallFiles.deleteFile)(blockKey);
-  log(`Remove block contents from S3...`);
-  await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
-  return block;
-}
-async function importTemplate({
-  templateKey,
-  context,
-  fileUploadsData
-}) {
-  const log = console.log;
-
-  // Making Directory for template in which we're going to extract the template data file.
-  const TEMPLATE_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, templateKey);
-  (0, _fsExtra.ensureDirSync)(TEMPLATE_EXTRACT_DIR);
-  const templateDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
-  const TEMPLATE_DATA_FILE_PATH = _path.default.join(TEMPLATE_EXTRACT_DIR, _path.default.basename(templateDataFileKey));
-  log(`Downloading Template data file: ${templateDataFileKey} at "${TEMPLATE_DATA_FILE_PATH}"`);
-  // Download and save template data file in disk.
-  await new Promise((resolve, reject) => {
-    _s3Stream.s3Stream.readStream(templateDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(TEMPLATE_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
-  });
-
-  // Load the template data file from disk.
-  log(`Load file ${templateDataFileKey}`);
-  const {
-    template,
-    files
-  } = await (0, _loadJsonFile.default)(TEMPLATE_DATA_FILE_PATH);
-
-  // Only update template data if there are files.
-  if (files && Array.isArray(files) && files.length > 0) {
-    // Upload template assets.
-    const {
-      fileIdToKeyMap
-    } = await uploadAssets({
-      context,
-      filesData: files,
-      fileUploadsData
-    });
-    const settings = await context.fileManager.settings.getSettings();
-    const {
-      srcPrefix = ""
-    } = settings || {};
-    updateFilesInData({
-      data: template.content || {},
-      fileIdToKeyMap,
-      srcPrefix
-    });
-  }
-  log("Removing Directory for template...");
-  await (0, _downloadInstallFiles.deleteFile)(templateKey);
-  log(`Remove template contents from S3...`);
-  await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
-  return template;
-}
-async function uploadFilesFromS3({
-  fileKeyToFileMap,
-  oldKeyToNewKeyMap
-}) {
-  const oldKeysForAssets = Object.keys(oldKeyToNewKeyMap);
-  const promises = [];
-  // Upload all assets.
-  for (let i = 0; i < oldKeysForAssets.length; i++) {
-    const oldKey = oldKeysForAssets[i];
-    const tempNewKey = oldKeyToNewKeyMap[oldKey];
-
-    // Read file.
-    const readStream = _s3Stream.s3Stream.readStream(tempNewKey);
-    // Get file meta data.
-    const fileMetaData = fileKeyToFileMap.get(oldKey);
-    if (fileMetaData) {
-      const newKey = (0, _uniqid.default)("", `-${fileMetaData.key}`);
-      const {
-        streamPassThrough,
-        streamPassThroughUploadPromise: promise
-      } = _s3Stream.s3Stream.writeStream(newKey, fileMetaData.type);
-      readStream.pipe(streamPassThrough);
-      promises.push(promise);
-      console.log(`Successfully queued file "${newKey}"`);
-    }
-  }
-  return Promise.all(promises);
-}
-function getOldFileKey(key) {
-  /*
-   * Because we know the naming convention, we can extract the old key from new key.
-   */
-  try {
-    const [, ...rest] = key.split("-");
-    return rest.join("-");
-  } catch (e) {
-    return key;
-  }
-}
-const FILE_CONTENT_TYPE = "application/octet-stream";
-function getFileNameWithoutExt(fileName) {
-  return _path.default.basename(fileName).replace(_path.default.extname(fileName), "");
-}
-/**
- * Function will read the given zip file from S3 via stream, extract its content and upload it to S3 bucket.
- * @param zipFileUrl
- * @return ImportData S3 file keys for all uploaded assets group by page/block.
- */
-async function readExtractAndUploadZipFileContents(zipFileUrl) {
-  const log = console.log;
-  const importDataList = [];
-  const zipFileName = _path.default.basename(zipFileUrl).split("?")[0];
-  const response = await (0, _nodeFetch.default)(zipFileUrl);
-  if (!response.ok) {
-    throw new _error.default(`Unable to downloading file: "${zipFileUrl}"`, response.statusText);
-  }
-  const readStream = response.body;
-  const uniquePath = (0, _uniqid.default)("IMPORTS/");
-  // Read export file and download it in the disk
-  const ZIP_FILE_PATH = _path.default.join(INSTALL_DIR, zipFileName);
-  const writeStream = (0, _fs.createWriteStream)(ZIP_FILE_PATH);
-  await streamPipeline(readStream, writeStream);
-  log(`Downloaded file "${zipFileName}" at ${ZIP_FILE_PATH}`);
-
-  // Extract the downloaded zip file
-  const zipFilePaths = await extractZipToDisk(ZIP_FILE_PATH);
-  log(`Removing ZIP file "${zipFileUrl}" from ${ZIP_FILE_PATH}`);
-  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH);
-
-  // Extract each page/block zip and upload their content's to S3
-  for (let i = 0; i < zipFilePaths.length; i++) {
-    const currentPath = zipFilePaths[i];
-    const dataMap = await extractZipAndUploadToS3(currentPath, uniquePath);
-    importDataList.push(dataMap);
-  }
-  log("Removing all ZIP files located at ", _path.default.dirname(zipFilePaths[0]));
-  await (0, _downloadInstallFiles.deleteFile)(_path.default.dirname(zipFilePaths[0]));
-  return importDataList;
-}
-const ASSETS_DIR_NAME = "/assets";
-function prepareDataDirMap({
-  map,
-  filePath,
-  newKey
-}) {
-  const dirname = _path.default.dirname(filePath);
-  const fileName = _path.default.basename(filePath);
-  /*
-   * We want to use dot (.) as part of object key rather than creating nested object(s).
-   * Also, the file name might contain dots in it beside the extension, so, we are escaping them all.
-   */
-  const oldKey = fileName.replace(/\./g, "\\.");
-  const isAsset = dirname.endsWith(ASSETS_DIR_NAME);
-  if (isAsset) {
-    map = _dotPropImmutable.default.set(map, `assets.${oldKey}`, newKey);
-  } else {
-    // We only need to know the newKey for data file.
-    map = _dotPropImmutable.default.set(map, `data`, newKey);
-  }
-  return map;
-}
-async function deleteS3Folder(key) {
-  // Append trailing slash i.e "/" to key to make sure we only delete a specific folder.
-  if (!key.endsWith("/")) {
-    key = `${key}/`;
-  }
-  const response = await _s3Stream.s3Stream.listObject(key);
-  const keys = (response.Contents || []).map(c => c.Key).filter(Boolean);
-  console.log(`Found ${keys.length} files.`);
-  const deleteFilePromises = keys.map(key => _s3Stream.s3Stream.deleteObject(key));
-  await Promise.all(deleteFilePromises);
-  console.log(`Successfully deleted ${deleteFilePromises.length} files.`);
-}
-
-// export const zeroPad = version => `${version}`.padStart(5, "0");
-
-function initialStats(total) {
-  return {
-    [_types.ImportExportTaskStatus.PENDING]: total,
-    [_types.ImportExportTaskStatus.PROCESSING]: 0,
-    [_types.ImportExportTaskStatus.COMPLETED]: 0,
-    [_types.ImportExportTaskStatus.FAILED]: 0,
-    total
-  };
-}
-function extractZipToDisk(exportFileZipPath) {
-  return new Promise((resolve, reject) => {
-    const zipFilePaths = [];
-    const uniqueFolderNameForExport = getFileNameWithoutExt(exportFileZipPath);
-    const EXPORT_FILE_EXTRACTION_PATH = _path.default.join(INSTALL_DIR, uniqueFolderNameForExport);
-    // Make sure DIR exists
-    (0, _fsExtra.ensureDirSync)(EXPORT_FILE_EXTRACTION_PATH);
-    _yauzl.default.open(exportFileZipPath, {
-      lazyEntries: true
-    }, function (err, zipFile) {
-      if (err) {
-        console.warn("ERROR: Failed to extract zip: ", exportFileZipPath, err);
-        reject(err);
-        return;
-      }
-      if (!zipFile) {
-        console.log("ERROR: Missing zip file resource for path: " + exportFileZipPath);
-        reject("Missing Zip File Resource.");
-        return;
-      }
-      console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
-      zipFile.on("end", function (err) {
-        if (err) {
-          console.warn("ERROR: Failed on END event for file: ", exportFileZipPath, err);
-          reject(err);
-        }
-        resolve(zipFilePaths);
-      });
-      zipFile.readEntry();
-      zipFile.on("entry", function (entry) {
-        console.info(`Processing entry: "${entry.fileName}"`);
-        if (/\/$/.test(entry.fileName)) {
-          // Directory file names end with '/'.
-          // Note that entries for directories themselves are optional.
-          // An entry's fileName implicitly requires its parent directories to exist.
-          zipFile.readEntry();
-        } else {
-          // file entry
-          zipFile.openReadStream(entry, function (err, readStream) {
-            if (err) {
-              console.warn("ERROR: Failed to openReadStream for file: ", entry.fileName, err);
-              reject(err);
-              return;
-            }
-            if (!readStream) {
-              console.log("ERROR: Missing Read Stream Resource when extracting to disk.");
-              reject("Missing Read Stream Resource.");
-              return;
-            }
-            const filePath = _path.default.join(EXPORT_FILE_EXTRACTION_PATH, entry.fileName);
-            readStream.on("end", function () {
-              zipFilePaths.push(filePath);
-              zipFile.readEntry();
-            });
-            streamPipeline(readStream, (0, _fs.createWriteStream)(filePath)).catch(error => {
-              reject(error);
-            });
-          });
-        }
-      });
-    });
-  });
-}
-function extractZipAndUploadToS3(dataZipFilePath, uniquePath) {
-  return new Promise((resolve, reject) => {
-    const filePaths = [];
-    const fileUploadPromises = [];
-    const uniqueKey = getFileNameWithoutExt(dataZipFilePath);
-    let dataMap = {
-      key: uniqueKey,
-      assets: {},
-      data: ""
-    };
-    _yauzl.default.open(dataZipFilePath, {
-      lazyEntries: true
-    }, function (err, zipFile) {
-      if (err) {
-        console.warn("ERROR: Failed to extract zip: ", dataZipFilePath, err);
-        reject(err);
-        return;
-      }
-      if (!zipFile) {
-        console.log("ERROR: Probably failed to extract zip: " + dataZipFilePath);
-        reject("Missing Zip File Resource.");
-        return;
-      }
-      console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
-      zipFile.on("end", function (err) {
-        if (err) {
-          console.warn('ERROR: Failed on "END" for file: ', dataZipFilePath, err);
-          reject(err);
-        }
-        Promise.all(fileUploadPromises).then(res => {
-          res.forEach(r => {
-            console.info("Done uploading... ", r);
-          });
-          resolve(dataMap);
-        });
-      });
-      zipFile.readEntry();
-      zipFile.on("entry", function (entry) {
-        console.info(`Processing entry: "${entry.fileName}"`);
-        if (/\/$/.test(entry.fileName)) {
-          // Directory file names end with '/'.
-          // Note that entries for directories themselves are optional.
-          // An entry's fileName implicitly requires its parent directories to exist.
-          zipFile.readEntry();
-        } else {
-          // file entry
-          zipFile.openReadStream(entry, function (err, readStream) {
-            if (err) {
-              console.warn("ERROR: Failed while performing [openReadStream] for file: ", entry.fileName, err);
-              reject(err);
-              return;
-            }
-            if (!readStream) {
-              console.log("ERROR: Missing Read Stream while importing.");
-              reject("Missing Read Strea Resource.");
-              return;
-            }
-            readStream.on("end", function () {
-              filePaths.push(entry.fileName);
-              zipFile.readEntry();
-            });
-            const newKey = `${uniquePath}/${uniqueKey}/${entry.fileName}`;
-            // Modify in place
-            dataMap = prepareDataDirMap({
-              map: dataMap,
-              filePath: entry.fileName,
-              newKey
-            });
-            const {
-              streamPassThrough,
-              streamPassThroughUploadPromise: promise
-            } = _s3Stream.s3Stream.writeStream(newKey, FILE_CONTENT_TYPE);
-            streamPipeline(readStream, streamPassThrough).then(() => {
-              fileUploadPromises.push(promise);
-            }).catch(error => {
-              reject(error);
-            });
-          });
-        }
-      });
-    });
-  });
-}