@webiny/api-page-builder-import-export 0.0.0-unstable.d16f688daf → 0.0.0-unstable.d65ec29d44

package/import/utils.js

@@ -1,53 +1,34 @@
 "use strict";
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.importBlock = importBlock;
 exports.importPage = importPage;
+exports.importTemplate = importTemplate;
 exports.initialStats = initialStats;
 exports.readExtractAndUploadZipFileContents = readExtractAndUploadZipFileContents;
 exports.uploadAssets = void 0;
-
 var _uniqid = _interopRequireDefault(require("uniqid"));
-
 var _dotPropImmutable = _interopRequireDefault(require("dot-prop-immutable"));
-
 var _fs = require("fs");
-
 var _fsExtra = require("fs-extra");
-
 var _util = require("util");
-
 var _stream = require("stream");
-
 var _nodeFetch = _interopRequireDefault(require("node-fetch"));
-
 var _path = _interopRequireDefault(require("path"));
-
 var _yauzl = _interopRequireDefault(require("yauzl"));
-
 var _chunk = _interopRequireDefault(require("lodash/chunk"));
-
 var _loadJsonFile = _interopRequireDefault(require("load-json-file"));
-
 var _error = _interopRequireDefault(require("@webiny/error"));
-
 var _downloadInstallFiles = require("@webiny/api-page-builder/graphql/crud/install/utils/downloadInstallFiles");
-
 var _types = require("../types");
-
 var _s3Stream = require("../export/s3Stream");
-
 const streamPipeline = (0, _util.promisify)(_stream.pipeline);
 const INSTALL_DIR = "/tmp";
-
 const INSTALL_EXTRACT_DIR = _path.default.join(INSTALL_DIR, "apiPageBuilderImport");
-
 const FILES_COUNT_IN_EACH_BATCH = 15;
-
 function updateImageInPageSettings(params) {
   const {
     settings,
@@ -56,22 +37,16 @@ function updateImageInPageSettings(params) {
   } = params;
   let newSettings = settings;
   const srcPrefixWithoutTrailingSlash = srcPrefix.endsWith("/") ? srcPrefix.slice(0, -1) : srcPrefix;
-
   if (_dotPropImmutable.default.get(newSettings, "general.image.src")) {
     var _settings$general, _settings$general$ima;
-
     newSettings = _dotPropImmutable.default.set(newSettings, "general.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$general = settings.general) === null || _settings$general === void 0 ? void 0 : (_settings$general$ima = _settings$general.image) === null || _settings$general$ima === void 0 ? void 0 : _settings$general$ima.id) || "")}`);
   }
-
   if (_dotPropImmutable.default.get(newSettings, "social.image.src")) {
     var _settings$social, _settings$social$imag;
-
     newSettings = _dotPropImmutable.default.set(newSettings, "social.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$social = settings.social) === null || _settings$social === void 0 ? void 0 : (_settings$social$imag = _settings$social.image) === null || _settings$social$imag === void 0 ? void 0 : _settings$social$imag.id) || "")}`);
   }
-
   return newSettings;
 }
-
 function updateBlockPreviewImage(params) {
   const {
     file,
@@ -83,7 +58,6 @@ function updateBlockPreviewImage(params) {
   newFile.src = `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(file.id || "")}`;
   return newFile;
 }
-
 function updateFilesInData({
   data,
   fileIdToKeyMap,
@@ -92,9 +66,8 @@ function updateFilesInData({
   // BASE CASE: Termination point
   if (!data || typeof data !== "object") {
     return;
-  } // Recursively call function if data is array
-
-
+  }
+  // Recursively call function if data is array
   if (Array.isArray(data)) {
     for (let i = 0; i < data.length; i++) {
       const element = data[i];
@@ -104,16 +77,12 @@ function updateFilesInData({
         srcPrefix
       });
     }
-
     return;
-  } // Main logic
-
-
+  }
+  // Main logic
   const tuple = Object.entries(data);
-
   for (let i = 0; i < tuple.length; i++) {
     const [key, value] = tuple[i];
-
     if (key === "file" && value && fileIdToKeyMap.has(value.id)) {
       value.key = fileIdToKeyMap.get(value.id);
       value.name = fileIdToKeyMap.get(value.id);
@@ -127,51 +96,49 @@ function updateFilesInData({
     }
   }
 }
-
 const uploadAssets = async params => {
   const {
     context,
     filesData,
     fileUploadsData
-  } = params; // Save uploaded file key against static id for later use.
-
+  } = params;
+  // Save uploaded file key against static id for later use.
   const fileIdToKeyMap = new Map();
   /**
    * This function contains logic of file download from S3.
    * Current we're not mocking zip file download from S3 in tests at the moment.
    * So, we're manually mocking it in case of test just by returning an empty object.
    */
-
   if (process.env.NODE_ENV === "test") {
     return {
       fileIdToKeyMap
     };
-  } // Save files meta data against old key for later use.
-
-
-  const fileKeyToFileMap = new Map(); // Initialize maps.
+  }
 
+  // Save files meta data against old key for later use.
+  const fileKeyToFileMap = new Map();
+  // Initialize maps.
   for (let i = 0; i < filesData.length; i++) {
     const file = filesData[i];
-    fileKeyToFileMap.set(file.key, file); // Initialize the value
+    fileKeyToFileMap.set(file.key, file);
 
+    // Initialize the value
     fileIdToKeyMap.set(file.id, file.type);
   }
-
   const fileUploadResults = await uploadFilesFromS3({
     fileKeyToFileMap,
     oldKeyToNewKeyMap: fileUploadsData.assets
-  }); // Create files in File Manager
+  });
 
+  // Create files in File Manager
   const createFilesInput = fileUploadResults.map(uploadResult => {
     const newKey = uploadResult.Key;
     const file = fileKeyToFileMap.get(getOldFileKey(newKey));
-
     if (!file) {
       return null;
-    } // Update the file map with newly uploaded file.
-
+    }
 
+    // Update the file map with newly uploaded file.
     fileIdToKeyMap.set(file.id, newKey);
     return {
       key: newKey,
@@ -182,10 +149,9 @@ const uploadAssets = async params => {
       tags: file.tags
     };
   }).filter(Boolean);
-  const createFilesPromises = []; // Gives an array of chunks (each consists of FILES_COUNT_IN_EACH_BATCH items).
-
+  const createFilesPromises = [];
+  // Gives an array of chunks (each consists of FILES_COUNT_IN_EACH_BATCH items).
   const createFilesInputChunks = (0, _chunk.default)(createFilesInput, FILES_COUNT_IN_EACH_BATCH);
-
   for (let i = 0; i < createFilesInputChunks.length; i++) {
     const createFilesInputChunk = createFilesInputChunks[i];
     createFilesPromises.push(
@@ -195,49 +161,44 @@ const uploadAssets = async params => {
      */
     context.fileManager.files.createFilesInBatch(createFilesInputChunk));
   }
-
   await Promise.all(createFilesPromises);
   return {
     fileIdToKeyMap
   };
 };
-
 exports.uploadAssets = uploadAssets;
-
 async function importPage({
   pageKey,
   context,
   fileUploadsData
 }) {
-  const log = console.log; // Making Directory for page in which we're going to extract the page data file.
+  const log = console.log;
 
+  // Making Directory for page in which we're going to extract the page data file.
   const PAGE_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, pageKey);
-
   (0, _fsExtra.ensureDirSync)(PAGE_EXTRACT_DIR);
-
   const pageDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
-
   const PAGE_DATA_FILE_PATH = _path.default.join(PAGE_EXTRACT_DIR, _path.default.basename(pageDataFileKey));
-
-  log(`Downloading Page data file: ${pageDataFileKey} at "${PAGE_DATA_FILE_PATH}"`); // Download and save page data file in disk.
-
+  log(`Downloading Page data file: ${pageDataFileKey} at "${PAGE_DATA_FILE_PATH}"`);
+  // Download and save page data file in disk.
   await new Promise((resolve, reject) => {
     _s3Stream.s3Stream.readStream(pageDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(PAGE_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
-  }); // Load the page data file from disk.
+  });
 
+  // Load the page data file from disk.
   log(`Load file ${pageDataFileKey}`);
   const {
     page,
     files
-  } = await (0, _loadJsonFile.default)(PAGE_DATA_FILE_PATH); // Only update page data if there are files.
+  } = await (0, _loadJsonFile.default)(PAGE_DATA_FILE_PATH);
 
+  // Only update page data if there are files.
   if (files && Array.isArray(files) && files.length > 0) {
     // Upload page assets.
     const {
       fileIdToKeyMap
     } = await uploadAssets({
       context,
-
       /**
        * TODO @ts-refactor @ashutosh figure out correct types.
        */
@@ -260,41 +221,38 @@ async function importPage({
       srcPrefix
     });
   }
-
   log("Removing Directory for page...");
   await (0, _downloadInstallFiles.deleteFile)(pageKey);
   log(`Remove page contents from S3...`);
   await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
   return page;
 }
-
 async function importBlock({
   blockKey,
   context,
   fileUploadsData
 }) {
-  const log = console.log; // Making Directory for block in which we're going to extract the block data file.
+  const log = console.log;
 
+  // Making Directory for block in which we're going to extract the block data file.
   const BLOCK_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, blockKey);
-
   (0, _fsExtra.ensureDirSync)(BLOCK_EXTRACT_DIR);
-
   const blockDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
-
   const BLOCK_DATA_FILE_PATH = _path.default.join(BLOCK_EXTRACT_DIR, _path.default.basename(blockDataFileKey));
-
-  log(`Downloading Block data file: ${blockDataFileKey} at "${BLOCK_DATA_FILE_PATH}"`); // Download and save block data file in disk.
-
+  log(`Downloading Block data file: ${blockDataFileKey} at "${BLOCK_DATA_FILE_PATH}"`);
+  // Download and save block data file in disk.
   await new Promise((resolve, reject) => {
     _s3Stream.s3Stream.readStream(blockDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(BLOCK_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
-  }); // Load the block data file from disk.
+  });
 
+  // Load the block data file from disk.
   log(`Load file ${blockDataFileKey}`);
   const {
     block,
     files
-  } = await (0, _loadJsonFile.default)(BLOCK_DATA_FILE_PATH); // Only update block data if there are files.
+  } = await (0, _loadJsonFile.default)(BLOCK_DATA_FILE_PATH);
 
+  // Only update block data if there are files.
   if (files && Array.isArray(files) && files.length > 0) {
     // Upload block assets.
     const {
@@ -319,47 +277,91 @@ async function importBlock({
       srcPrefix
     });
   }
-
   log("Removing Directory for block...");
   await (0, _downloadInstallFiles.deleteFile)(blockKey);
   log(`Remove block contents from S3...`);
   await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
   return block;
 }
+async function importTemplate({
+  templateKey,
+  context,
+  fileUploadsData
+}) {
+  const log = console.log;
 
+  // Making Directory for template in which we're going to extract the template data file.
+  const TEMPLATE_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, templateKey);
+  (0, _fsExtra.ensureDirSync)(TEMPLATE_EXTRACT_DIR);
+  const templateDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
+  const TEMPLATE_DATA_FILE_PATH = _path.default.join(TEMPLATE_EXTRACT_DIR, _path.default.basename(templateDataFileKey));
+  log(`Downloading Template data file: ${templateDataFileKey} at "${TEMPLATE_DATA_FILE_PATH}"`);
+  // Download and save template data file in disk.
+  await new Promise((resolve, reject) => {
+    _s3Stream.s3Stream.readStream(templateDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(TEMPLATE_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
+  });
+
+  // Load the template data file from disk.
+  log(`Load file ${templateDataFileKey}`);
+  const {
+    template,
+    files
+  } = await (0, _loadJsonFile.default)(TEMPLATE_DATA_FILE_PATH);
+
+  // Only update template data if there are files.
+  if (files && Array.isArray(files) && files.length > 0) {
+    // Upload template assets.
+    const {
+      fileIdToKeyMap
+    } = await uploadAssets({
+      context,
+      filesData: files,
+      fileUploadsData
+    });
+    const settings = await context.fileManager.settings.getSettings();
+    const {
+      srcPrefix = ""
+    } = settings || {};
+    updateFilesInData({
+      data: template.content || {},
+      fileIdToKeyMap,
+      srcPrefix
+    });
+  }
+  log("Removing Directory for template...");
+  await (0, _downloadInstallFiles.deleteFile)(templateKey);
+  log(`Remove template contents from S3...`);
+  await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
+  return template;
+}
 async function uploadFilesFromS3({
   fileKeyToFileMap,
   oldKeyToNewKeyMap
 }) {
   const oldKeysForAssets = Object.keys(oldKeyToNewKeyMap);
-  const promises = []; // Upload all assets.
-
+  const promises = [];
+  // Upload all assets.
   for (let i = 0; i < oldKeysForAssets.length; i++) {
     const oldKey = oldKeysForAssets[i];
-    const tempNewKey = oldKeyToNewKeyMap[oldKey]; // Read file.
-
-    const readStream = _s3Stream.s3Stream.readStream(tempNewKey); // Get file meta data.
-
+    const tempNewKey = oldKeyToNewKeyMap[oldKey];
 
+    // Read file.
+    const readStream = _s3Stream.s3Stream.readStream(tempNewKey);
+    // Get file meta data.
     const fileMetaData = fileKeyToFileMap.get(oldKey);
-
     if (fileMetaData) {
       const newKey = (0, _uniqid.default)("", `-${fileMetaData.key}`);
-
       const {
         streamPassThrough,
         streamPassThroughUploadPromise: promise
       } = _s3Stream.s3Stream.writeStream(newKey, fileMetaData.type);
-
       readStream.pipe(streamPassThrough);
       promises.push(promise);
       console.log(`Successfully queued file "${newKey}"`);
     }
   }
-
   return Promise.all(promises);
 }
-
 function getOldFileKey(key) {
   /*
    * Because we know the naming convention, we can extract the old key from new key.
@@ -371,13 +373,10 @@ function getOldFileKey(key) {
     return key;
   }
 }
-
 const FILE_CONTENT_TYPE = "application/octet-stream";
-
 function getFileNameWithoutExt(fileName) {
   return _path.default.basename(fileName).replace(_path.default.extname(fileName), "");
 }
-
 /**
  * Function will read the given zip file from S3 via stream, extract its content and upload it to S3 bucket.
  * @param zipFileUrl
@@ -386,82 +385,70 @@ function getFileNameWithoutExt(fileName) {
 async function readExtractAndUploadZipFileContents(zipFileUrl) {
   const log = console.log;
   const importDataList = [];
-
   const zipFileName = _path.default.basename(zipFileUrl).split("?")[0];
-
   const response = await (0, _nodeFetch.default)(zipFileUrl);
-
   if (!response.ok) {
     throw new _error.default(`Unable to downloading file: "${zipFileUrl}"`, response.statusText);
   }
-
   const readStream = response.body;
-  const uniquePath = (0, _uniqid.default)("IMPORTS/"); // Read export file and download it in the disk
-
+  const uniquePath = (0, _uniqid.default)("IMPORTS/");
+  // Read export file and download it in the disk
   const ZIP_FILE_PATH = _path.default.join(INSTALL_DIR, zipFileName);
-
   const writeStream = (0, _fs.createWriteStream)(ZIP_FILE_PATH);
   await streamPipeline(readStream, writeStream);
-  log(`Downloaded file "${zipFileName}" at ${ZIP_FILE_PATH}`); // Extract the downloaded zip file
+  log(`Downloaded file "${zipFileName}" at ${ZIP_FILE_PATH}`);
 
+  // Extract the downloaded zip file
   const zipFilePaths = await extractZipToDisk(ZIP_FILE_PATH);
   log(`Removing ZIP file "${zipFileUrl}" from ${ZIP_FILE_PATH}`);
-  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH); // Extract each page/block zip and upload their content's to S3
+  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH);
 
+  // Extract each page/block zip and upload their content's to S3
   for (let i = 0; i < zipFilePaths.length; i++) {
     const currentPath = zipFilePaths[i];
     const dataMap = await extractZipAndUploadToS3(currentPath, uniquePath);
     importDataList.push(dataMap);
   }
-
   log("Removing all ZIP files located at ", _path.default.dirname(zipFilePaths[0]));
   await (0, _downloadInstallFiles.deleteFile)(_path.default.dirname(zipFilePaths[0]));
   return importDataList;
 }
-
 const ASSETS_DIR_NAME = "/assets";
-
 function prepareDataDirMap({
   map,
   filePath,
   newKey
 }) {
   const dirname = _path.default.dirname(filePath);
-
   const fileName = _path.default.basename(filePath);
   /*
    * We want to use dot (.) as part of object key rather than creating nested object(s).
    * Also, the file name might contain dots in it beside the extension, so, we are escaping them all.
    */
-
-
   const oldKey = fileName.replace(/\./g, "\\.");
   const isAsset = dirname.endsWith(ASSETS_DIR_NAME);
-
   if (isAsset) {
     map = _dotPropImmutable.default.set(map, `assets.${oldKey}`, newKey);
   } else {
     // We only need to know the newKey for data file.
     map = _dotPropImmutable.default.set(map, `data`, newKey);
   }
-
   return map;
 }
-
 async function deleteS3Folder(key) {
   // Append trailing slash i.e "/" to key to make sure we only delete a specific folder.
   if (!key.endsWith("/")) {
     key = `${key}/`;
   }
-
   const response = await _s3Stream.s3Stream.listObject(key);
   const keys = (response.Contents || []).map(c => c.Key).filter(Boolean);
   console.log(`Found ${keys.length} files.`);
   const deleteFilePromises = keys.map(key => _s3Stream.s3Stream.deleteObject(key));
   await Promise.all(deleteFilePromises);
   console.log(`Successfully deleted ${deleteFilePromises.length} files.`);
-} // export const zeroPad = version => `${version}`.padStart(5, "0");
+}
 
+// export const zeroPad = version => `${version}`.padStart(5, "0");
 
 function initialStats(total) {
   return {
@@ -472,17 +459,13 @@ function initialStats(total) {
     total
   };
 }
-
 function extractZipToDisk(exportFileZipPath) {
   return new Promise((resolve, reject) => {
     const zipFilePaths = [];
     const uniqueFolderNameForExport = getFileNameWithoutExt(exportFileZipPath);
-
-    const EXPORT_FILE_EXTRACTION_PATH = _path.default.join(INSTALL_DIR, uniqueFolderNameForExport); // Make sure DIR exists
-
-
+    const EXPORT_FILE_EXTRACTION_PATH = _path.default.join(INSTALL_DIR, uniqueFolderNameForExport);
+    // Make sure DIR exists
     (0, _fsExtra.ensureDirSync)(EXPORT_FILE_EXTRACTION_PATH);
-
     _yauzl.default.open(exportFileZipPath, {
       lazyEntries: true
     }, function (err, zipFile) {
@@ -491,26 +474,22 @@ function extractZipToDisk(exportFileZipPath) {
         reject(err);
         return;
       }
-
       if (!zipFile) {
         console.log("ERROR: Missing zip file resource for path: " + exportFileZipPath);
         reject("Missing Zip File Resource.");
         return;
       }
-
       console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
       zipFile.on("end", function (err) {
         if (err) {
           console.warn("ERROR: Failed on END event for file: ", exportFileZipPath, err);
           reject(err);
         }
-
         resolve(zipFilePaths);
       });
       zipFile.readEntry();
       zipFile.on("entry", function (entry) {
         console.info(`Processing entry: "${entry.fileName}"`);
-
         if (/\/$/.test(entry.fileName)) {
           // Directory file names end with '/'.
           // Note that entries for directories themselves are optional.
@@ -524,15 +503,12 @@ function extractZipToDisk(exportFileZipPath) {
               reject(err);
               return;
             }
-
             if (!readStream) {
               console.log("ERROR: Missing Read Stream Resource when extracting to disk.");
               reject("Missing Read Stream Resource.");
               return;
             }
-
             const filePath = _path.default.join(EXPORT_FILE_EXTRACTION_PATH, entry.fileName);
-
             readStream.on("end", function () {
               zipFilePaths.push(filePath);
               zipFile.readEntry();
@@ -546,7 +522,6 @@ function extractZipToDisk(exportFileZipPath) {
     });
   });
 }
-
 function extractZipAndUploadToS3(dataZipFilePath, uniquePath) {
   return new Promise((resolve, reject) => {
     const filePaths = [];
@@ -557,7 +532,6 @@ function extractZipAndUploadToS3(dataZipFilePath, uniquePath) {
       assets: {},
       data: ""
     };
-
     _yauzl.default.open(dataZipFilePath, {
       lazyEntries: true
     }, function (err, zipFile) {
@@ -566,20 +540,17 @@ function extractZipAndUploadToS3(dataZipFilePath, uniquePath) {
         reject(err);
         return;
       }
-
       if (!zipFile) {
         console.log("ERROR: Probably failed to extract zip: " + dataZipFilePath);
         reject("Missing Zip File Resource.");
         return;
       }
-
       console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
       zipFile.on("end", function (err) {
         if (err) {
           console.warn('ERROR: Failed on "END" for file: ', dataZipFilePath, err);
           reject(err);
         }
-
         Promise.all(fileUploadPromises).then(res => {
           res.forEach(r => {
             console.info("Done uploading... ", r);
@@ -590,7 +561,6 @@ function extractZipAndUploadToS3(dataZipFilePath, uniquePath) {
       zipFile.readEntry();
       zipFile.on("entry", function (entry) {
         console.info(`Processing entry: "${entry.fileName}"`);
-
         if (/\/$/.test(entry.fileName)) {
           // Directory file names end with '/'.
           // Note that entries for directories themselves are optional.
@@ -604,30 +574,26 @@ function extractZipAndUploadToS3(dataZipFilePath, uniquePath) {
               reject(err);
               return;
             }
-
             if (!readStream) {
               console.log("ERROR: Missing Read Stream while importing.");
               reject("Missing Read Strea Resource.");
               return;
             }
-
             readStream.on("end", function () {
               filePaths.push(entry.fileName);
               zipFile.readEntry();
             });
-            const newKey = `${uniquePath}/${uniqueKey}/${entry.fileName}`; // Modify in place
-
+            const newKey = `${uniquePath}/${uniqueKey}/${entry.fileName}`;
+            // Modify in place
             dataMap = prepareDataDirMap({
               map: dataMap,
               filePath: entry.fileName,
               newKey
             });
-
             const {
               streamPassThrough,
               streamPassThroughUploadPromise: promise
             } = _s3Stream.s3Stream.writeStream(newKey, FILE_CONTENT_TYPE);
-
             streamPipeline(readStream, streamPassThrough).then(() => {
               fileUploadPromises.push(promise);
             }).catch(error => {