@webiny/api-page-builder-import-export 0.0.0-unstable.1e66d121db → 0.0.0-unstable.611c5af35e

package/importPages/utils.js

@@ -1,7 +1,6 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
@@ -9,44 +8,25 @@ exports.importPage = importPage;
 exports.initialStats = initialStats;
 exports.readExtractAndUploadZipFileContents = readExtractAndUploadZipFileContents;
 exports.uploadPageAssets = void 0;
-
 var _uniqid = _interopRequireDefault(require("uniqid"));
-
 var _dotPropImmutable = _interopRequireDefault(require("dot-prop-immutable"));
-
 var _fs = require("fs");
-
 var _fsExtra = require("fs-extra");
-
 var _util = require("util");
-
 var _stream = require("stream");
-
 var _nodeFetch = _interopRequireDefault(require("node-fetch"));
-
 var _path = _interopRequireDefault(require("path"));
-
 var _yauzl = _interopRequireDefault(require("yauzl"));
-
 var _chunk = _interopRequireDefault(require("lodash/chunk"));
-
 var _loadJsonFile = _interopRequireDefault(require("load-json-file"));
-
 var _error = _interopRequireDefault(require("@webiny/error"));
-
 var _downloadInstallFiles = require("@webiny/api-page-builder/graphql/crud/install/utils/downloadInstallFiles");
-
 var _types = require("../types");
-
 var _s3Stream = require("../exportPages/s3Stream");
-
 const streamPipeline = (0, _util.promisify)(_stream.pipeline);
 const INSTALL_DIR = "/tmp";
-
 const INSTALL_EXTRACT_DIR = _path.default.join(INSTALL_DIR, "apiPageBuilderImportPage");
-
 const FILES_COUNT_IN_EACH_BATCH = 15;
-
 function updateImageInPageSettings(params) {
   const {
     settings,
@@ -55,22 +35,16 @@ function updateImageInPageSettings(params) {
   } = params;
   let newSettings = settings;
   const srcPrefixWithoutTrailingSlash = srcPrefix.endsWith("/") ? srcPrefix.slice(0, -1) : srcPrefix;
-
   if (_dotPropImmutable.default.get(newSettings, "general.image.src")) {
     var _settings$general, _settings$general$ima;
-
     newSettings = _dotPropImmutable.default.set(newSettings, "general.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$general = settings.general) === null || _settings$general === void 0 ? void 0 : (_settings$general$ima = _settings$general.image) === null || _settings$general$ima === void 0 ? void 0 : _settings$general$ima.id) || "")}`);
   }
-
   if (_dotPropImmutable.default.get(newSettings, "social.image.src")) {
     var _settings$social, _settings$social$imag;
-
     newSettings = _dotPropImmutable.default.set(newSettings, "social.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$social = settings.social) === null || _settings$social === void 0 ? void 0 : (_settings$social$imag = _settings$social.image) === null || _settings$social$imag === void 0 ? void 0 : _settings$social$imag.id) || "")}`);
   }
-
   return newSettings;
 }
-
 function updateFilesInPageData({
   data,
   fileIdToKeyMap,
@@ -79,9 +53,8 @@ function updateFilesInPageData({
   // BASE CASE: Termination point
   if (!data || typeof data !== "object") {
     return;
-  } // Recursively call function if data is array
-
-
+  }
+  // Recursively call function if data is array
   if (Array.isArray(data)) {
     for (let i = 0; i < data.length; i++) {
       const element = data[i];
@@ -91,16 +64,12 @@ function updateFilesInPageData({
         srcPrefix
       });
     }
-
     return;
-  } // Main logic
-
-
+  }
+  // Main logic
   const tuple = Object.entries(data);
-
   for (let i = 0; i < tuple.length; i++) {
     const [key, value] = tuple[i];
-
     if (key === "file" && value && fileIdToKeyMap.has(value.id)) {
       value.key = fileIdToKeyMap.get(value.id);
       value.name = fileIdToKeyMap.get(value.id);
@@ -114,52 +83,50 @@ function updateFilesInPageData({
     }
   }
 }
-
 const uploadPageAssets = async params => {
   const {
     context,
     filesData,
     fileUploadsData
-  } = params; // Save uploaded file key against static id for later use.
-
+  } = params;
+  // Save uploaded file key against static id for later use.
   const fileIdToKeyMap = new Map();
   /**
    * This function contains logic of file download from S3.
    * Current we're not mocking zip file download from S3 in tests at the moment.
    * So, we're manually mocking it in case of test just by returning an empty object.
    */
-
   if (process.env.NODE_ENV === "test") {
     return {
       fileIdToKeyMap
     };
   }
+  console.log("INSIDE uploadPageAssets");
 
-  console.log("INSIDE uploadPageAssets"); // Save files meta data against old key for later use.
-
-  const fileKeyToFileMap = new Map(); // Initialize maps.
-
+  // Save files meta data against old key for later use.
+  const fileKeyToFileMap = new Map();
+  // Initialize maps.
   for (let i = 0; i < filesData.length; i++) {
     const file = filesData[i];
-    fileKeyToFileMap.set(file.key, file); // Initialize the value
+    fileKeyToFileMap.set(file.key, file);
 
+    // Initialize the value
     fileIdToKeyMap.set(file.id, file.type);
   }
-
   const fileUploadResults = await uploadFilesFromS3({
     fileKeyToFileMap,
     oldKeyToNewKeyMap: fileUploadsData.assets
-  }); // Create files in File Manager
+  });
 
+  // Create files in File Manager
   const createFilesInput = fileUploadResults.map(uploadResult => {
     const newKey = uploadResult.Key;
     const file = fileKeyToFileMap.get(getOldFileKey(newKey));
-
     if (!file) {
       return null;
-    } // Update the file map with newly uploaded file.
-
+    }
 
+    // Update the file map with newly uploaded file.
     fileIdToKeyMap.set(file.id, newKey);
     return {
       key: newKey,
@@ -170,10 +137,9 @@ const uploadPageAssets = async params => {
       tags: file.tags
     };
   }).filter(Boolean);
-  const createFilesPromises = []; // Gives an array of chunks (each consists of FILES_COUNT_IN_EACH_BATCH items).
-
+  const createFilesPromises = [];
+  // Gives an array of chunks (each consists of FILES_COUNT_IN_EACH_BATCH items).
   const createFilesInputChunks = (0, _chunk.default)(createFilesInput, FILES_COUNT_IN_EACH_BATCH);
-
   for (let i = 0; i < createFilesInputChunks.length; i++) {
     const createFilesInputChunk = createFilesInputChunks[i];
     createFilesPromises.push(
@@ -183,49 +149,44 @@ const uploadPageAssets = async params => {
      */
     context.fileManager.files.createFilesInBatch(createFilesInputChunk));
   }
-
   await Promise.all(createFilesPromises);
   return {
     fileIdToKeyMap
   };
 };
-
 exports.uploadPageAssets = uploadPageAssets;
-
 async function importPage({
   pageKey,
   context,
   fileUploadsData
 }) {
-  const log = console.log; // Making Directory for page in which we're going to extract the page data file.
+  const log = console.log;
 
+  // Making Directory for page in which we're going to extract the page data file.
   const PAGE_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, pageKey);
-
   (0, _fsExtra.ensureDirSync)(PAGE_EXTRACT_DIR);
-
   const pageDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
-
   const PAGE_DATA_FILE_PATH = _path.default.join(PAGE_EXTRACT_DIR, _path.default.basename(pageDataFileKey));
-
-  log(`Downloading Page data file: ${pageDataFileKey} at "${PAGE_DATA_FILE_PATH}"`); // Download and save page data file in disk.
-
+  log(`Downloading Page data file: ${pageDataFileKey} at "${PAGE_DATA_FILE_PATH}"`);
+  // Download and save page data file in disk.
   await new Promise((resolve, reject) => {
     _s3Stream.s3Stream.readStream(pageDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(PAGE_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
-  }); // Load the page data file from disk.
+  });
 
+  // Load the page data file from disk.
   log(`Load file ${pageDataFileKey}`);
   const {
     page,
     files
-  } = await (0, _loadJsonFile.default)(PAGE_DATA_FILE_PATH); // Only update page data if there are files.
+  } = await (0, _loadJsonFile.default)(PAGE_DATA_FILE_PATH);
 
+  // Only update page data if there are files.
   if (files && Array.isArray(files) && files.length > 0) {
     // Upload page assets.
     const {
       fileIdToKeyMap
     } = await uploadPageAssets({
       context,
-
       /**
        * TODO @ts-refactor @ashutosh figure out correct types.
        */
@@ -248,47 +209,40 @@ async function importPage({
       srcPrefix
     });
   }
-
   log("Removing Directory for page...");
   await (0, _downloadInstallFiles.deleteFile)(pageKey);
   log(`Remove page contents from S3...`);
   await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
   return page;
 }
-
 async function uploadFilesFromS3({
   fileKeyToFileMap,
   oldKeyToNewKeyMap
 }) {
   const oldKeysForAssets = Object.keys(oldKeyToNewKeyMap);
-  const promises = []; // Upload all assets.
-
+  const promises = [];
+  // Upload all assets.
   for (let i = 0; i < oldKeysForAssets.length; i++) {
     const oldKey = oldKeysForAssets[i];
-    const tempNewKey = oldKeyToNewKeyMap[oldKey]; // Read file.
-
-    const readStream = _s3Stream.s3Stream.readStream(tempNewKey); // Get file meta data.
-
+    const tempNewKey = oldKeyToNewKeyMap[oldKey];
 
+    // Read file.
+    const readStream = _s3Stream.s3Stream.readStream(tempNewKey);
+    // Get file meta data.
     const fileMetaData = fileKeyToFileMap.get(oldKey);
-
     if (fileMetaData) {
       const newKey = (0, _uniqid.default)("", `-${fileMetaData.key}`);
-
       const {
         streamPassThrough,
         streamPassThroughUploadPromise: promise
       } = _s3Stream.s3Stream.writeStream(newKey, fileMetaData.type);
-
       readStream.pipe(streamPassThrough);
       promises.push(promise);
       console.log(`Successfully queued file "${newKey}"`);
     }
   }
-
   return Promise.all(promises);
 }
-
 function getOldFileKey(key) {
   /*
    * Because we know the naming convention, we can extract the old key from new key.
@@ -300,13 +254,10 @@ function getOldFileKey(key) {
     return key;
   }
 }
-
 const FILE_CONTENT_TYPE = "application/octet-stream";
-
 function getFileNameWithoutExt(fileName) {
   return _path.default.basename(fileName).replace(_path.default.extname(fileName), "");
 }
-
 /**
  * Function will read the given zip file from S3 via stream, extract its content and upload it to S3 bucket.
  * @param zipFileUrl
@@ -315,82 +266,70 @@ function getFileNameWithoutExt(fileName) {
 async function readExtractAndUploadZipFileContents(zipFileUrl) {
   const log = console.log;
   const pageImportDataList = [];
-
   const zipFileName = _path.default.basename(zipFileUrl).split("?")[0];
-
   const response = await (0, _nodeFetch.default)(zipFileUrl);
-
   if (!response.ok) {
     throw new _error.default(`Unable to downloading file: "${zipFileUrl}"`, response.statusText);
   }
-
   const readStream = response.body;
-  const uniquePath = (0, _uniqid.default)("IMPORT_PAGES/"); // Read export file and download it in the disk
-
+  const uniquePath = (0, _uniqid.default)("IMPORT_PAGES/");
+  // Read export file and download it in the disk
   const ZIP_FILE_PATH = _path.default.join(INSTALL_DIR, zipFileName);
-
   const writeStream = (0, _fs.createWriteStream)(ZIP_FILE_PATH);
   await streamPipeline(readStream, writeStream);
-  log(`Downloaded file "${zipFileName}" at ${ZIP_FILE_PATH}`); // Extract the downloaded zip file
+  log(`Downloaded file "${zipFileName}" at ${ZIP_FILE_PATH}`);
 
+  // Extract the downloaded zip file
   const zipFilePaths = await extractZipToDisk(ZIP_FILE_PATH);
   log(`Removing ZIP file "${zipFileUrl}" from ${ZIP_FILE_PATH}`);
-  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH); // Extract each page zip and upload their content's to S3
+  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH);
 
+  // Extract each page zip and upload their content's to S3
   for (let i = 0; i < zipFilePaths.length; i++) {
     const currentPath = zipFilePaths[i];
     const dataMap = await extractZipAndUploadToS3(currentPath, uniquePath);
     pageImportDataList.push(dataMap);
   }
-
   log("Removing all ZIP files located at ", _path.default.dirname(zipFilePaths[0]));
   await (0, _downloadInstallFiles.deleteFile)(_path.default.dirname(zipFilePaths[0]));
   return pageImportDataList;
 }
-
 const ASSETS_DIR_NAME = "/assets";
-
 function preparePageDataDirMap({
   map,
   filePath,
   newKey
 }) {
   const dirname = _path.default.dirname(filePath);
-
   const fileName = _path.default.basename(filePath);
   /*
    * We want to use dot (.) as part of object key rather than creating nested object(s).
    * Also, the file name might contain dots in it beside the extension, so, we are escaping them all.
    */
-
-
   const oldKey = fileName.replace(/\./g, "\\.");
   const isAsset = dirname.endsWith(ASSETS_DIR_NAME);
-
   if (isAsset) {
     map = _dotPropImmutable.default.set(map, `assets.${oldKey}`, newKey);
   } else {
     // We only need to know the newKey for data file.
     map = _dotPropImmutable.default.set(map, `data`, newKey);
   }
-
   return map;
 }
-
 async function deleteS3Folder(key) {
   // Append trailing slash i.e "/" to key to make sure we only delete a specific folder.
   if (!key.endsWith("/")) {
     key = `${key}/`;
   }
-
   const response = await _s3Stream.s3Stream.listObject(key);
   const keys = (response.Contents || []).map(c => c.Key).filter(Boolean);
   console.log(`Found ${keys.length} files.`);
   const deleteFilePromises = keys.map(key => _s3Stream.s3Stream.deleteObject(key));
   await Promise.all(deleteFilePromises);
   console.log(`Successfully deleted ${deleteFilePromises.length} files.`);
-} // export const zeroPad = version => `${version}`.padStart(5, "0");
+}
 
+// export const zeroPad = version => `${version}`.padStart(5, "0");
 
 function initialStats(total) {
   return {
@@ -401,17 +340,13 @@ function initialStats(total) {
     total
   };
 }
-
 function extractZipToDisk(exportFileZipPath) {
   return new Promise((resolve, reject) => {
     const pageZipFilePaths = [];
     const uniqueFolderNameForExport = getFileNameWithoutExt(exportFileZipPath);
-
-    const EXPORT_FILE_EXTRACTION_PATH = _path.default.join(INSTALL_DIR, uniqueFolderNameForExport); // Make sure DIR exists
-
-
+    const EXPORT_FILE_EXTRACTION_PATH = _path.default.join(INSTALL_DIR, uniqueFolderNameForExport);
+    // Make sure DIR exists
     (0, _fsExtra.ensureDirSync)(EXPORT_FILE_EXTRACTION_PATH);
-
     _yauzl.default.open(exportFileZipPath, {
       lazyEntries: true
     }, function (err, zipFile) {
@@ -420,26 +355,22 @@ function extractZipToDisk(exportFileZipPath) {
         reject(err);
         return;
       }
-
       if (!zipFile) {
         console.log("ERROR: Missing zip file resource for path: " + exportFileZipPath);
         reject("Missing Zip File Resource.");
         return;
       }
-
       console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
       zipFile.on("end", function (err) {
         if (err) {
           console.warn("ERROR: Failed on END event for file: ", exportFileZipPath, err);
           reject(err);
         }
-
         resolve(pageZipFilePaths);
       });
       zipFile.readEntry();
       zipFile.on("entry", function (entry) {
         console.info(`Processing entry: "${entry.fileName}"`);
-
         if (/\/$/.test(entry.fileName)) {
           // Directory file names end with '/'.
           // Note that entries for directories themselves are optional.
@@ -453,15 +384,12 @@ function extractZipToDisk(exportFileZipPath) {
               reject(err);
               return;
             }
-
             if (!readStream) {
               console.log("ERROR: Missing Read Stream Resource when extracting to disk.");
               reject("Missing Read Stream Resource.");
               return;
             }
-
             const filePath = _path.default.join(EXPORT_FILE_EXTRACTION_PATH, entry.fileName);
-
             readStream.on("end", function () {
               pageZipFilePaths.push(filePath);
               zipFile.readEntry();
@@ -475,7 +403,6 @@ function extractZipToDisk(exportFileZipPath) {
     });
   });
 }
-
 function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
   return new Promise((resolve, reject) => {
     const filePaths = [];
@@ -486,7 +413,6 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
       assets: {},
       data: ""
     };
-
     _yauzl.default.open(pageDataZipFilePath, {
       lazyEntries: true
     }, function (err, zipFile) {
@@ -495,20 +421,17 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
         reject(err);
         return;
       }
-
       if (!zipFile) {
         console.log("ERROR: Probably failed to extract zip: " + pageDataZipFilePath);
         reject("Missing Zip File Resource.");
         return;
       }
-
       console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
       zipFile.on("end", function (err) {
         if (err) {
           console.warn('ERROR: Failed on "END" for file: ', pageDataZipFilePath, err);
           reject(err);
         }
-
         Promise.all(fileUploadPromises).then(res => {
           res.forEach(r => {
             console.info("Done uploading... ", r);
@@ -519,7 +442,6 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
       zipFile.readEntry();
       zipFile.on("entry", function (entry) {
         console.info(`Processing entry: "${entry.fileName}"`);
-
         if (/\/$/.test(entry.fileName)) {
           // Directory file names end with '/'.
           // Note that entries for directories themselves are optional.
@@ -533,30 +455,26 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
               reject(err);
               return;
             }
-
             if (!readStream) {
               console.log("ERROR: Missing Read Stream while importing pages.");
               reject("Missing Read Strea Resource.");
               return;
             }
-
             readStream.on("end", function () {
               filePaths.push(entry.fileName);
               zipFile.readEntry();
             });
-            const newKey = `${uniquePath}/${uniquePageKey}/${entry.fileName}`; // Modify in place
-
+            const newKey = `${uniquePath}/${uniquePageKey}/${entry.fileName}`;
+            // Modify in place
             dataMap = preparePageDataDirMap({
               map: dataMap,
               filePath: entry.fileName,
               newKey
             });
-
             const {
               streamPassThrough,
               streamPassThroughUploadPromise: promise
             } = _s3Stream.s3Stream.writeStream(newKey, FILE_CONTENT_TYPE);
-
             streamPipeline(readStream, streamPassThrough).then(() => {
               fileUploadPromises.push(promise);
             }).catch(error => {