@webiny/api-page-builder-import-export 0.0.0-unstable.990c3ab1b6 → 0.0.0-unstable.d4f203fa97

package/importPages/utils.js

@@ -1,6 +1,7 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
+
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
@@ -8,25 +9,44 @@ exports.importPage = importPage;
 exports.initialStats = initialStats;
 exports.readExtractAndUploadZipFileContents = readExtractAndUploadZipFileContents;
 exports.uploadPageAssets = void 0;
+
 var _uniqid = _interopRequireDefault(require("uniqid"));
+
 var _dotPropImmutable = _interopRequireDefault(require("dot-prop-immutable"));
+
 var _fs = require("fs");
+
 var _fsExtra = require("fs-extra");
+
 var _util = require("util");
+
 var _stream = require("stream");
+
 var _nodeFetch = _interopRequireDefault(require("node-fetch"));
+
 var _path = _interopRequireDefault(require("path"));
+
 var _yauzl = _interopRequireDefault(require("yauzl"));
+
 var _chunk = _interopRequireDefault(require("lodash/chunk"));
+
 var _loadJsonFile = _interopRequireDefault(require("load-json-file"));
+
 var _error = _interopRequireDefault(require("@webiny/error"));
+
 var _downloadInstallFiles = require("@webiny/api-page-builder/graphql/crud/install/utils/downloadInstallFiles");
+
 var _types = require("../types");
+
 var _s3Stream = require("../exportPages/s3Stream");
+
 const streamPipeline = (0, _util.promisify)(_stream.pipeline);
 const INSTALL_DIR = "/tmp";
+
 const INSTALL_EXTRACT_DIR = _path.default.join(INSTALL_DIR, "apiPageBuilderImportPage");
+
 const FILES_COUNT_IN_EACH_BATCH = 15;
+
 function updateImageInPageSettings(params) {
   const {
     settings,
@@ -35,16 +55,22 @@ function updateImageInPageSettings(params) {
   } = params;
   let newSettings = settings;
   const srcPrefixWithoutTrailingSlash = srcPrefix.endsWith("/") ? srcPrefix.slice(0, -1) : srcPrefix;
+
   if (_dotPropImmutable.default.get(newSettings, "general.image.src")) {
     var _settings$general, _settings$general$ima;
+
     newSettings = _dotPropImmutable.default.set(newSettings, "general.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$general = settings.general) === null || _settings$general === void 0 ? void 0 : (_settings$general$ima = _settings$general.image) === null || _settings$general$ima === void 0 ? void 0 : _settings$general$ima.id) || "")}`);
   }
+
   if (_dotPropImmutable.default.get(newSettings, "social.image.src")) {
     var _settings$social, _settings$social$imag;
+
     newSettings = _dotPropImmutable.default.set(newSettings, "social.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$social = settings.social) === null || _settings$social === void 0 ? void 0 : (_settings$social$imag = _settings$social.image) === null || _settings$social$imag === void 0 ? void 0 : _settings$social$imag.id) || "")}`);
   }
+
   return newSettings;
 }
+
 function updateFilesInPageData({
   data,
   fileIdToKeyMap,
@@ -53,8 +79,9 @@ function updateFilesInPageData({
   // BASE CASE: Termination point
   if (!data || typeof data !== "object") {
     return;
-  }
-  // Recursively call function if data is array
+  } // Recursively call function if data is array
+
+
   if (Array.isArray(data)) {
     for (let i = 0; i < data.length; i++) {
       const element = data[i];
@@ -64,12 +91,16 @@ function updateFilesInPageData({
         srcPrefix
       });
     }
+
     return;
-  }
-  // Main logic
+  } // Main logic
+
+
   const tuple = Object.entries(data);
+
   for (let i = 0; i < tuple.length; i++) {
     const [key, value] = tuple[i];
+
     if (key === "file" && value && fileIdToKeyMap.has(value.id)) {
       value.key = fileIdToKeyMap.get(value.id);
       value.name = fileIdToKeyMap.get(value.id);
@@ -83,50 +114,52 @@ function updateFilesInPageData({
     }
   }
 }
+
 const uploadPageAssets = async params => {
   const {
     context,
     filesData,
     fileUploadsData
-  } = params;
-  // Save uploaded file key against static id for later use.
+  } = params; // Save uploaded file key against static id for later use.
+
   const fileIdToKeyMap = new Map();
   /**
    * This function contains logic of file download from S3.
    * Current we're not mocking zip file download from S3 in tests at the moment.
    * So, we're manually mocking it in case of test just by returning an empty object.
    */
+
   if (process.env.NODE_ENV === "test") {
     return {
       fileIdToKeyMap
     };
   }
-  console.log("INSIDE uploadPageAssets");
 
-  // Save files meta data against old key for later use.
-  const fileKeyToFileMap = new Map();
-  // Initialize maps.
+  console.log("INSIDE uploadPageAssets"); // Save files meta data against old key for later use.
+
+  const fileKeyToFileMap = new Map(); // Initialize maps.
+
   for (let i = 0; i < filesData.length; i++) {
     const file = filesData[i];
-    fileKeyToFileMap.set(file.key, file);
+    fileKeyToFileMap.set(file.key, file); // Initialize the value
 
-    // Initialize the value
     fileIdToKeyMap.set(file.id, file.type);
   }
+
   const fileUploadResults = await uploadFilesFromS3({
     fileKeyToFileMap,
     oldKeyToNewKeyMap: fileUploadsData.assets
-  });
+  }); // Create files in File Manager
 
-  // Create files in File Manager
   const createFilesInput = fileUploadResults.map(uploadResult => {
     const newKey = uploadResult.Key;
     const file = fileKeyToFileMap.get(getOldFileKey(newKey));
+
     if (!file) {
       return null;
-    }
+    } // Update the file map with newly uploaded file.
+
 
-    // Update the file map with newly uploaded file.
     fileIdToKeyMap.set(file.id, newKey);
     return {
       key: newKey,
@@ -137,9 +170,10 @@ const uploadPageAssets = async params => {
       tags: file.tags
     };
   }).filter(Boolean);
-  const createFilesPromises = [];
-  // Gives an array of chunks (each consists of FILES_COUNT_IN_EACH_BATCH items).
+  const createFilesPromises = []; // Gives an array of chunks (each consists of FILES_COUNT_IN_EACH_BATCH items).
+
   const createFilesInputChunks = (0, _chunk.default)(createFilesInput, FILES_COUNT_IN_EACH_BATCH);
+
   for (let i = 0; i < createFilesInputChunks.length; i++) {
     const createFilesInputChunk = createFilesInputChunks[i];
     createFilesPromises.push(
@@ -149,44 +183,49 @@ const uploadPageAssets = async params => {
      */
     context.fileManager.files.createFilesInBatch(createFilesInputChunk));
   }
+
   await Promise.all(createFilesPromises);
   return {
     fileIdToKeyMap
   };
 };
+
 exports.uploadPageAssets = uploadPageAssets;
+
 async function importPage({
   pageKey,
   context,
   fileUploadsData
 }) {
-  const log = console.log;
+  const log = console.log; // Making Directory for page in which we're going to extract the page data file.
 
-  // Making Directory for page in which we're going to extract the page data file.
   const PAGE_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, pageKey);
+
   (0, _fsExtra.ensureDirSync)(PAGE_EXTRACT_DIR);
+
   const pageDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
+
   const PAGE_DATA_FILE_PATH = _path.default.join(PAGE_EXTRACT_DIR, _path.default.basename(pageDataFileKey));
-  log(`Downloading Page data file: ${pageDataFileKey} at "${PAGE_DATA_FILE_PATH}"`);
-  // Download and save page data file in disk.
+
+  log(`Downloading Page data file: ${pageDataFileKey} at "${PAGE_DATA_FILE_PATH}"`); // Download and save page data file in disk.
+
   await new Promise((resolve, reject) => {
     _s3Stream.s3Stream.readStream(pageDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(PAGE_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
-  });
+  }); // Load the page data file from disk.
 
-  // Load the page data file from disk.
   log(`Load file ${pageDataFileKey}`);
   const {
     page,
     files
-  } = await (0, _loadJsonFile.default)(PAGE_DATA_FILE_PATH);
+  } = await (0, _loadJsonFile.default)(PAGE_DATA_FILE_PATH); // Only update page data if there are files.
 
-  // Only update page data if there are files.
   if (files && Array.isArray(files) && files.length > 0) {
     // Upload page assets.
     const {
       fileIdToKeyMap
     } = await uploadPageAssets({
       context,
+
       /**
        * TODO @ts-refactor @ashutosh figure out correct types.
        */
@@ -209,40 +248,47 @@ async function importPage({
       srcPrefix
     });
   }
+
   log("Removing Directory for page...");
   await (0, _downloadInstallFiles.deleteFile)(pageKey);
   log(`Remove page contents from S3...`);
   await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
   return page;
 }
+
 async function uploadFilesFromS3({
   fileKeyToFileMap,
   oldKeyToNewKeyMap
 }) {
   const oldKeysForAssets = Object.keys(oldKeyToNewKeyMap);
-  const promises = [];
-  // Upload all assets.
+  const promises = []; // Upload all assets.
+
   for (let i = 0; i < oldKeysForAssets.length; i++) {
     const oldKey = oldKeysForAssets[i];
-    const tempNewKey = oldKeyToNewKeyMap[oldKey];
+    const tempNewKey = oldKeyToNewKeyMap[oldKey]; // Read file.
+
+    const readStream = _s3Stream.s3Stream.readStream(tempNewKey); // Get file meta data.
+
 
-    // Read file.
-    const readStream = _s3Stream.s3Stream.readStream(tempNewKey);
-    // Get file meta data.
     const fileMetaData = fileKeyToFileMap.get(oldKey);
+
     if (fileMetaData) {
       const newKey = (0, _uniqid.default)("", `-${fileMetaData.key}`);
+
       const {
         streamPassThrough,
         streamPassThroughUploadPromise: promise
       } = _s3Stream.s3Stream.writeStream(newKey, fileMetaData.type);
+
       readStream.pipe(streamPassThrough);
       promises.push(promise);
       console.log(`Successfully queued file "${newKey}"`);
     }
   }
+
   return Promise.all(promises);
 }
+
 function getOldFileKey(key) {
   /*
    * Because we know the naming convention, we can extract the old key from new key.
@@ -254,10 +300,13 @@ function getOldFileKey(key) {
     return key;
   }
 }
+
 const FILE_CONTENT_TYPE = "application/octet-stream";
+
 function getFileNameWithoutExt(fileName) {
   return _path.default.basename(fileName).replace(_path.default.extname(fileName), "");
 }
+
 /**
  * Function will read the given zip file from S3 via stream, extract its content and upload it to S3 bucket.
  * @param zipFileUrl
@@ -266,70 +315,82 @@ function getFileNameWithoutExt(fileName) {
 async function readExtractAndUploadZipFileContents(zipFileUrl) {
   const log = console.log;
   const pageImportDataList = [];
+
   const zipFileName = _path.default.basename(zipFileUrl).split("?")[0];
+
   const response = await (0, _nodeFetch.default)(zipFileUrl);
+
   if (!response.ok) {
     throw new _error.default(`Unable to downloading file: "${zipFileUrl}"`, response.statusText);
   }
+
   const readStream = response.body;
-  const uniquePath = (0, _uniqid.default)("IMPORT_PAGES/");
-  // Read export file and download it in the disk
+  const uniquePath = (0, _uniqid.default)("IMPORT_PAGES/"); // Read export file and download it in the disk
+
   const ZIP_FILE_PATH = _path.default.join(INSTALL_DIR, zipFileName);
+
   const writeStream = (0, _fs.createWriteStream)(ZIP_FILE_PATH);
   await streamPipeline(readStream, writeStream);
-  log(`Downloaded file "${zipFileName}" at ${ZIP_FILE_PATH}`);
+  log(`Downloaded file "${zipFileName}" at ${ZIP_FILE_PATH}`); // Extract the downloaded zip file
 
-  // Extract the downloaded zip file
   const zipFilePaths = await extractZipToDisk(ZIP_FILE_PATH);
   log(`Removing ZIP file "${zipFileUrl}" from ${ZIP_FILE_PATH}`);
-  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH);
+  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH); // Extract each page zip and upload their content's to S3
 
-  // Extract each page zip and upload their content's to S3
   for (let i = 0; i < zipFilePaths.length; i++) {
     const currentPath = zipFilePaths[i];
     const dataMap = await extractZipAndUploadToS3(currentPath, uniquePath);
     pageImportDataList.push(dataMap);
   }
+
   log("Removing all ZIP files located at ", _path.default.dirname(zipFilePaths[0]));
   await (0, _downloadInstallFiles.deleteFile)(_path.default.dirname(zipFilePaths[0]));
   return pageImportDataList;
 }
+
 const ASSETS_DIR_NAME = "/assets";
+
 function preparePageDataDirMap({
   map,
   filePath,
   newKey
 }) {
   const dirname = _path.default.dirname(filePath);
+
   const fileName = _path.default.basename(filePath);
   /*
    * We want to use dot (.) as part of object key rather than creating nested object(s).
    * Also, the file name might contain dots in it beside the extension, so, we are escaping them all.
    */
+
+
   const oldKey = fileName.replace(/\./g, "\\.");
   const isAsset = dirname.endsWith(ASSETS_DIR_NAME);
+
   if (isAsset) {
     map = _dotPropImmutable.default.set(map, `assets.${oldKey}`, newKey);
   } else {
     // We only need to know the newKey for data file.
     map = _dotPropImmutable.default.set(map, `data`, newKey);
   }
+
   return map;
 }
+
 async function deleteS3Folder(key) {
   // Append trailing slash i.e "/" to key to make sure we only delete a specific folder.
   if (!key.endsWith("/")) {
     key = `${key}/`;
   }
+
   const response = await _s3Stream.s3Stream.listObject(key);
   const keys = (response.Contents || []).map(c => c.Key).filter(Boolean);
   console.log(`Found ${keys.length} files.`);
   const deleteFilePromises = keys.map(key => _s3Stream.s3Stream.deleteObject(key));
   await Promise.all(deleteFilePromises);
   console.log(`Successfully deleted ${deleteFilePromises.length} files.`);
-}
+} // export const zeroPad = version => `${version}`.padStart(5, "0");
 
-// export const zeroPad = version => `${version}`.padStart(5, "0");
 
 function initialStats(total) {
   return {
@@ -340,13 +401,17 @@ function initialStats(total) {
     total
   };
 }
+
 function extractZipToDisk(exportFileZipPath) {
   return new Promise((resolve, reject) => {
     const pageZipFilePaths = [];
     const uniqueFolderNameForExport = getFileNameWithoutExt(exportFileZipPath);
-    const EXPORT_FILE_EXTRACTION_PATH = _path.default.join(INSTALL_DIR, uniqueFolderNameForExport);
-    // Make sure DIR exists
+
+    const EXPORT_FILE_EXTRACTION_PATH = _path.default.join(INSTALL_DIR, uniqueFolderNameForExport); // Make sure DIR exists
+
+
     (0, _fsExtra.ensureDirSync)(EXPORT_FILE_EXTRACTION_PATH);
+
     _yauzl.default.open(exportFileZipPath, {
       lazyEntries: true
     }, function (err, zipFile) {
@@ -355,22 +420,26 @@ function extractZipToDisk(exportFileZipPath) {
         reject(err);
         return;
       }
+
       if (!zipFile) {
         console.log("ERROR: Missing zip file resource for path: " + exportFileZipPath);
         reject("Missing Zip File Resource.");
         return;
       }
+
       console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
       zipFile.on("end", function (err) {
         if (err) {
           console.warn("ERROR: Failed on END event for file: ", exportFileZipPath, err);
           reject(err);
         }
+
         resolve(pageZipFilePaths);
       });
       zipFile.readEntry();
       zipFile.on("entry", function (entry) {
         console.info(`Processing entry: "${entry.fileName}"`);
+
         if (/\/$/.test(entry.fileName)) {
           // Directory file names end with '/'.
           // Note that entries for directories themselves are optional.
@@ -384,12 +453,15 @@ function extractZipToDisk(exportFileZipPath) {
               reject(err);
               return;
             }
+
             if (!readStream) {
               console.log("ERROR: Missing Read Stream Resource when extracting to disk.");
               reject("Missing Read Stream Resource.");
               return;
             }
+
             const filePath = _path.default.join(EXPORT_FILE_EXTRACTION_PATH, entry.fileName);
+
             readStream.on("end", function () {
               pageZipFilePaths.push(filePath);
               zipFile.readEntry();
@@ -403,6 +475,7 @@ function extractZipToDisk(exportFileZipPath) {
     });
   });
 }
+
 function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
   return new Promise((resolve, reject) => {
     const filePaths = [];
@@ -413,6 +486,7 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
       assets: {},
       data: ""
     };
+
     _yauzl.default.open(pageDataZipFilePath, {
       lazyEntries: true
     }, function (err, zipFile) {
@@ -421,17 +495,20 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
         reject(err);
         return;
       }
+
       if (!zipFile) {
         console.log("ERROR: Probably failed to extract zip: " + pageDataZipFilePath);
         reject("Missing Zip File Resource.");
         return;
       }
+
       console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
       zipFile.on("end", function (err) {
         if (err) {
           console.warn('ERROR: Failed on "END" for file: ', pageDataZipFilePath, err);
           reject(err);
         }
+
         Promise.all(fileUploadPromises).then(res => {
           res.forEach(r => {
             console.info("Done uploading... ", r);
@@ -442,6 +519,7 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
       zipFile.readEntry();
       zipFile.on("entry", function (entry) {
         console.info(`Processing entry: "${entry.fileName}"`);
+
         if (/\/$/.test(entry.fileName)) {
           // Directory file names end with '/'.
           // Note that entries for directories themselves are optional.
@@ -455,26 +533,30 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
               reject(err);
               return;
             }
+
             if (!readStream) {
               console.log("ERROR: Missing Read Stream while importing pages.");
               reject("Missing Read Strea Resource.");
               return;
             }
+
             readStream.on("end", function () {
               filePaths.push(entry.fileName);
               zipFile.readEntry();
             });
-            const newKey = `${uniquePath}/${uniquePageKey}/${entry.fileName}`;
-            // Modify in place
+            const newKey = `${uniquePath}/${uniquePageKey}/${entry.fileName}`; // Modify in place
+
             dataMap = preparePageDataDirMap({
               map: dataMap,
               filePath: entry.fileName,
               newKey
             });
+
             const {
               streamPassThrough,
               streamPassThroughUploadPromise: promise
             } = _s3Stream.s3Stream.writeStream(newKey, FILE_CONTENT_TYPE);
+
             streamPipeline(readStream, streamPassThrough).then(() => {
               fileUploadPromises.push(promise);
             }).catch(error => {