@webiny/api-page-builder-import-export 0.0.0-mt-3 → 0.0.0-unstable.3386f66516

package/{importPages → import}/utils.js

@@ -1,14 +1,16 @@
 "use strict";
 
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
 
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
+exports.importBlock = importBlock;
 exports.importPage = importPage;
+exports.importTemplate = importTemplate;
 exports.initialStats = initialStats;
 exports.readExtractAndUploadZipFileContents = readExtractAndUploadZipFileContents;
-exports.zeroPad = exports.uploadPageAssets = void 0;
+exports.uploadAssets = void 0;
 
 var _uniqid = _interopRequireDefault(require("uniqid"));
 
@@ -38,36 +40,52 @@ var _downloadInstallFiles = require("@webiny/api-page-builder/graphql/crud/insta
 
 var _types = require("../types");
 
-var _s3Stream = require("../exportPages/s3Stream");
+var _s3Stream = require("../export/s3Stream");
 
 const streamPipeline = (0, _util.promisify)(_stream.pipeline);
 const INSTALL_DIR = "/tmp";
 
-const INSTALL_EXTRACT_DIR = _path.default.join(INSTALL_DIR, "apiPageBuilderImportPage");
+const INSTALL_EXTRACT_DIR = _path.default.join(INSTALL_DIR, "apiPageBuilderImport");
 
 const FILES_COUNT_IN_EACH_BATCH = 15;
-const ZIP_CONTENT_TYPE = "application/zip";
 
-function updateImageInPageSettings({
-  settings,
-  fileIdToKeyMap,
-  srcPrefix
-}) {
+function updateImageInPageSettings(params) {
+  const {
+    settings,
+    fileIdToKeyMap,
+    srcPrefix
+  } = params;
   let newSettings = settings;
   const srcPrefixWithoutTrailingSlash = srcPrefix.endsWith("/") ? srcPrefix.slice(0, -1) : srcPrefix;
 
   if (_dotPropImmutable.default.get(newSettings, "general.image.src")) {
-    newSettings = _dotPropImmutable.default.set(newSettings, "general.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(settings.general.image.id)}`);
+    var _settings$general, _settings$general$ima;
+
+    newSettings = _dotPropImmutable.default.set(newSettings, "general.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$general = settings.general) === null || _settings$general === void 0 ? void 0 : (_settings$general$ima = _settings$general.image) === null || _settings$general$ima === void 0 ? void 0 : _settings$general$ima.id) || "")}`);
   }
 
   if (_dotPropImmutable.default.get(newSettings, "social.image.src")) {
-    newSettings = _dotPropImmutable.default.set(newSettings, "social.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(settings.social.image.id)}`);
+    var _settings$social, _settings$social$imag;
+
+    newSettings = _dotPropImmutable.default.set(newSettings, "social.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(((_settings$social = settings.social) === null || _settings$social === void 0 ? void 0 : (_settings$social$imag = _settings$social.image) === null || _settings$social$imag === void 0 ? void 0 : _settings$social$imag.id) || "")}`);
   }
 
-  return settings;
+  return newSettings;
+}
+
+function updateBlockPreviewImage(params) {
+  const {
+    file,
+    fileIdToKeyMap,
+    srcPrefix
+  } = params;
+  const newFile = file;
+  const srcPrefixWithoutTrailingSlash = srcPrefix.endsWith("/") ? srcPrefix.slice(0, -1) : srcPrefix;
+  newFile.src = `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(file.id || "")}`;
+  return newFile;
 }
 
-function updateFilesInPageData({
+function updateFilesInData({
   data,
   fileIdToKeyMap,
   srcPrefix
@@ -81,7 +99,7 @@ function updateFilesInPageData({
   if (Array.isArray(data)) {
     for (let i = 0; i < data.length; i++) {
       const element = data[i];
-      updateFilesInPageData({
+      updateFilesInData({
         data: element,
         fileIdToKeyMap,
         srcPrefix
@@ -102,7 +120,7 @@ function updateFilesInPageData({
       value.name = fileIdToKeyMap.get(value.id);
       value.src = `${srcPrefix}${srcPrefix.endsWith("/") ? "" : "/"}${fileIdToKeyMap.get(value.id)}`;
     } else {
-      updateFilesInPageData({
+      updateFilesInData({
         data: value,
         srcPrefix,
         fileIdToKeyMap
@@ -111,23 +129,26 @@ function updateFilesInPageData({
   }
 }
 
-const uploadPageAssets = async ({
-  context,
-  filesData,
-  fileUploadsData
-}) => {
+const uploadAssets = async params => {
+  const {
+    context,
+    filesData,
+    fileUploadsData
+  } = params; // Save uploaded file key against static id for later use.
+
+  const fileIdToKeyMap = new Map();
   /**
    * This function contains logic of file download from S3.
    * Current we're not mocking zip file download from S3 in tests at the moment.
    * So, we're manually mocking it in case of test just by returning an empty object.
    */
-  if (process.env.NODE_ENV === "test") {
-    return {};
-  }
 
-  console.log("INSIDE uploadPageAssets"); // Save uploaded file key against static id for later use.
+  if (process.env.NODE_ENV === "test") {
+    return {
+      fileIdToKeyMap
+    };
+  } // Save files meta data against old key for later use.
 
-  const fileIdToKeyMap = new Map(); // Save files meta data against old key for later use.
 
   const fileKeyToFileMap = new Map(); // Initialize maps.
 
@@ -145,7 +166,12 @@ const uploadPageAssets = async ({
 
   const createFilesInput = fileUploadResults.map(uploadResult => {
     const newKey = uploadResult.Key;
-    const file = fileKeyToFileMap.get(getOldFileKey(newKey)); // Update the file map with newly uploaded file.
+    const file = fileKeyToFileMap.get(getOldFileKey(newKey));
+
+    if (!file) {
+      return null;
+    } // Update the file map with newly uploaded file.
+
 
     fileIdToKeyMap.set(file.id, newKey);
     return {
@@ -156,7 +182,7 @@ const uploadPageAssets = async ({
       meta: file.meta,
       tags: file.tags
     };
-  });
+  }).filter(Boolean);
   const createFilesPromises = []; // Gives an array of chunks (each consists of FILES_COUNT_IN_EACH_BATCH items).
 
   const createFilesInputChunks = (0, _chunk.default)(createFilesInput, FILES_COUNT_IN_EACH_BATCH);
@@ -177,7 +203,7 @@ const uploadPageAssets = async ({
   };
 };
 
-exports.uploadPageAssets = uploadPageAssets;
+exports.uploadAssets = uploadAssets;
 
 async function importPage({
   pageKey,
@@ -206,25 +232,31 @@ async function importPage({
     files
   } = await (0, _loadJsonFile.default)(PAGE_DATA_FILE_PATH); // Only update page data if there are files.
 
-  if (Array.isArray(files) && files.length) {
+  if (files && Array.isArray(files) && files.length > 0) {
     // Upload page assets.
     const {
       fileIdToKeyMap
-    } = await uploadPageAssets({
+    } = await uploadAssets({
       context,
+
+      /**
+       * TODO @ts-refactor @ashutosh figure out correct types.
+       */
+      // @ts-ignore
       filesData: files,
       fileUploadsData
     });
+    const settings = await context.fileManager.settings.getSettings();
     const {
-      srcPrefix
-    } = await context.fileManager.settings.getSettings();
-    updateFilesInPageData({
-      data: page.content,
+      srcPrefix = ""
+    } = settings || {};
+    updateFilesInData({
+      data: page.content || {},
       fileIdToKeyMap,
       srcPrefix
     });
     page.settings = updateImageInPageSettings({
-      settings: page.settings,
+      settings: page.settings || {},
       fileIdToKeyMap,
       srcPrefix
     });
@@ -237,6 +269,119 @@ async function importPage({
   return page;
 }
 
+async function importBlock({
+  blockKey,
+  context,
+  fileUploadsData
+}) {
+  const log = console.log; // Making Directory for block in which we're going to extract the block data file.
+
+  const BLOCK_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, blockKey);
+
+  (0, _fsExtra.ensureDirSync)(BLOCK_EXTRACT_DIR);
+
+  const blockDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
+
+  const BLOCK_DATA_FILE_PATH = _path.default.join(BLOCK_EXTRACT_DIR, _path.default.basename(blockDataFileKey));
+
+  log(`Downloading Block data file: ${blockDataFileKey} at "${BLOCK_DATA_FILE_PATH}"`); // Download and save block data file in disk.
+
+  await new Promise((resolve, reject) => {
+    _s3Stream.s3Stream.readStream(blockDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(BLOCK_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
+  }); // Load the block data file from disk.
+
+  log(`Load file ${blockDataFileKey}`);
+  const {
+    block,
+    files
+  } = await (0, _loadJsonFile.default)(BLOCK_DATA_FILE_PATH); // Only update block data if there are files.
+
+  if (files && Array.isArray(files) && files.length > 0) {
+    // Upload block assets.
+    const {
+      fileIdToKeyMap
+    } = await uploadAssets({
+      context,
+      filesData: files,
+      fileUploadsData
+    });
+    const settings = await context.fileManager.settings.getSettings();
+    const {
+      srcPrefix = ""
+    } = settings || {};
+    updateFilesInData({
+      data: block.content || {},
+      fileIdToKeyMap,
+      srcPrefix
+    });
+    block.preview = updateBlockPreviewImage({
+      file: block.preview || {},
+      fileIdToKeyMap,
+      srcPrefix
+    });
+  }
+
+  log("Removing Directory for block...");
+  await (0, _downloadInstallFiles.deleteFile)(blockKey);
+  log(`Remove block contents from S3...`);
+  await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
+  return block;
+}
+
+async function importTemplate({
+  templateKey,
+  context,
+  fileUploadsData
+}) {
+  const log = console.log; // Making Directory for template in which we're going to extract the template data file.
+
+  const TEMPLATE_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, templateKey);
+
+  (0, _fsExtra.ensureDirSync)(TEMPLATE_EXTRACT_DIR);
+
+  const templateDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
+
+  const TEMPLATE_DATA_FILE_PATH = _path.default.join(TEMPLATE_EXTRACT_DIR, _path.default.basename(templateDataFileKey));
+
+  log(`Downloading Template data file: ${templateDataFileKey} at "${TEMPLATE_DATA_FILE_PATH}"`); // Download and save template data file in disk.
+
+  await new Promise((resolve, reject) => {
+    _s3Stream.s3Stream.readStream(templateDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(TEMPLATE_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
+  }); // Load the template data file from disk.
+
+  log(`Load file ${templateDataFileKey}`);
+  const {
+    template,
+    files
+  } = await (0, _loadJsonFile.default)(TEMPLATE_DATA_FILE_PATH); // Only update template data if there are files.
+
+  if (files && Array.isArray(files) && files.length > 0) {
+    // Upload template assets.
+    const {
+      fileIdToKeyMap
+    } = await uploadAssets({
+      context,
+      filesData: files,
+      fileUploadsData
+    });
+    const settings = await context.fileManager.settings.getSettings();
+    const {
+      srcPrefix = ""
+    } = settings || {};
+    updateFilesInData({
+      data: template.content || {},
+      fileIdToKeyMap,
+      srcPrefix
+    });
+  }
+
+  log("Removing Directory for template...");
+  await (0, _downloadInstallFiles.deleteFile)(templateKey);
+  log(`Remove template contents from S3...`);
+  await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
+  return template;
+}
+
 async function uploadFilesFromS3({
   fileKeyToFileMap,
   oldKeyToNewKeyMap
@@ -270,14 +415,6 @@ async function uploadFilesFromS3({
   return Promise.all(promises);
 }
 
-async function getObjectMetaFromS3(Key) {
-  const meta = await _s3Stream.s3Stream.getObjectHead(Key);
-
-  if (meta.ContentType !== ZIP_CONTENT_TYPE) {
-    throw new _error.default(`Unsupported file type: "${meta.ContentType}"`, "UNSUPPORTED_FILE");
-  }
-}
-
 function getOldFileKey(key) {
   /*
    * Because we know the naming convention, we can extract the old key from new key.
@@ -298,32 +435,23 @@ function getFileNameWithoutExt(fileName) {
 
 /**
  * Function will read the given zip file from S3 via stream, extract its content and upload it to S3 bucket.
- * @param zipFileKey
- * @return PageImportData S3 file keys for all uploaded assets group by page.
+ * @param zipFileUrl
+ * @return ImportData S3 file keys for all uploaded assets group by page/block.
  */
-async function readExtractAndUploadZipFileContents(zipFileKey) {
+async function readExtractAndUploadZipFileContents(zipFileUrl) {
   const log = console.log;
-  const pageImportDataList = [];
-  let readStream; // Check whether it is a URL
+  const importDataList = [];
 
-  if (zipFileKey.startsWith("http")) {
-    const response = await (0, _nodeFetch.default)(zipFileKey);
+  const zipFileName = _path.default.basename(zipFileUrl).split("?")[0];
 
-    if (!response.ok) {
-      throw new _error.default(`Unable to downloading file: "${zipFileKey}"`, response.statusText);
-    }
+  const response = await (0, _nodeFetch.default)(zipFileUrl);
 
-    readStream = response.body;
-  } else {
-    // We're first retrieving object's meta data, just to check whether the file is available at the given Key
-    await getObjectMetaFromS3(zipFileKey);
-    readStream = _s3Stream.s3Stream.readStream(zipFileKey);
+  if (!response.ok) {
+    throw new _error.default(`Unable to downloading file: "${zipFileUrl}"`, response.statusText);
   }
 
-  const uniquePath = (0, _uniqid.default)("IMPORT_PAGES/");
-
-  const zipFileName = _path.default.basename(zipFileKey); // Read export file and download it in the disk
-
+  const readStream = response.body;
+  const uniquePath = (0, _uniqid.default)("IMPORTS/"); // Read export file and download it in the disk
 
   const ZIP_FILE_PATH = _path.default.join(INSTALL_DIR, zipFileName);
 
@@ -332,23 +460,23 @@ async function readExtractAndUploadZipFileContents(zipFileKey) {
   log(`Downloaded file "${zipFileName}" at ${ZIP_FILE_PATH}`); // Extract the downloaded zip file
 
   const zipFilePaths = await extractZipToDisk(ZIP_FILE_PATH);
-  log(`Removing ZIP file "${zipFileKey}" from ${ZIP_FILE_PATH}`);
-  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH); // Extract each page zip and upload their content's to S3
+  log(`Removing ZIP file "${zipFileUrl}" from ${ZIP_FILE_PATH}`);
+  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH); // Extract each page/block zip and upload their content's to S3
 
   for (let i = 0; i < zipFilePaths.length; i++) {
     const currentPath = zipFilePaths[i];
     const dataMap = await extractZipAndUploadToS3(currentPath, uniquePath);
-    pageImportDataList.push(dataMap);
+    importDataList.push(dataMap);
   }
 
   log("Removing all ZIP files located at ", _path.default.dirname(zipFilePaths[0]));
   await (0, _downloadInstallFiles.deleteFile)(_path.default.dirname(zipFilePaths[0]));
-  return pageImportDataList;
+  return importDataList;
 }
 
 const ASSETS_DIR_NAME = "/assets";
 
-function preparePageDataDirMap({
+function prepareDataDirMap({
   map,
   filePath,
   newKey
@@ -382,30 +510,27 @@ async function deleteS3Folder(key) {
   }
 
   const response = await _s3Stream.s3Stream.listObject(key);
-  const keys = response.Contents.map(c => c.Key);
+  const keys = (response.Contents || []).map(c => c.Key).filter(Boolean);
   console.log(`Found ${keys.length} files.`);
   const deleteFilePromises = keys.map(key => _s3Stream.s3Stream.deleteObject(key));
   await Promise.all(deleteFilePromises);
   console.log(`Successfully deleted ${deleteFilePromises.length} files.`);
-}
-
-const zeroPad = version => `${version}`.padStart(5, "0");
+} // export const zeroPad = version => `${version}`.padStart(5, "0");
 
-exports.zeroPad = zeroPad;
 
 function initialStats(total) {
   return {
-    [_types.PageImportExportTaskStatus.PENDING]: total,
-    [_types.PageImportExportTaskStatus.PROCESSING]: 0,
-    [_types.PageImportExportTaskStatus.COMPLETED]: 0,
-    [_types.PageImportExportTaskStatus.FAILED]: 0,
+    [_types.ImportExportTaskStatus.PENDING]: total,
+    [_types.ImportExportTaskStatus.PROCESSING]: 0,
+    [_types.ImportExportTaskStatus.COMPLETED]: 0,
+    [_types.ImportExportTaskStatus.FAILED]: 0,
     total
   };
 }
 
 function extractZipToDisk(exportFileZipPath) {
   return new Promise((resolve, reject) => {
-    const pageZipFilePaths = [];
+    const zipFilePaths = [];
     const uniqueFolderNameForExport = getFileNameWithoutExt(exportFileZipPath);
 
     const EXPORT_FILE_EXTRACTION_PATH = _path.default.join(INSTALL_DIR, uniqueFolderNameForExport); // Make sure DIR exists
@@ -419,6 +544,13 @@ function extractZipToDisk(exportFileZipPath) {
       if (err) {
         console.warn("ERROR: Failed to extract zip: ", exportFileZipPath, err);
         reject(err);
+        return;
+      }
+
+      if (!zipFile) {
+        console.log("ERROR: Missing zip file resource for path: " + exportFileZipPath);
+        reject("Missing Zip File Resource.");
+        return;
       }
 
       console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
@@ -428,7 +560,7 @@ function extractZipToDisk(exportFileZipPath) {
           reject(err);
         }
 
-        resolve(pageZipFilePaths);
+        resolve(zipFilePaths);
       });
       zipFile.readEntry();
       zipFile.on("entry", function (entry) {
@@ -445,15 +577,24 @@ function extractZipToDisk(exportFileZipPath) {
             if (err) {
               console.warn("ERROR: Failed to openReadStream for file: ", entry.fileName, err);
               reject(err);
+              return;
+            }
+
+            if (!readStream) {
+              console.log("ERROR: Missing Read Stream Resource when extracting to disk.");
+              reject("Missing Read Stream Resource.");
+              return;
             }
 
             const filePath = _path.default.join(EXPORT_FILE_EXTRACTION_PATH, entry.fileName);
 
             readStream.on("end", function () {
-              pageZipFilePaths.push(filePath);
+              zipFilePaths.push(filePath);
               zipFile.readEntry();
             });
-            streamPipeline(readStream, (0, _fs.createWriteStream)(filePath));
+            streamPipeline(readStream, (0, _fs.createWriteStream)(filePath)).catch(error => {
+              reject(error);
+            });
           });
         }
       });
@@ -461,29 +602,36 @@ function extractZipToDisk(exportFileZipPath) {
   });
 }
 
-function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
+function extractZipAndUploadToS3(dataZipFilePath, uniquePath) {
   return new Promise((resolve, reject) => {
     const filePaths = [];
     const fileUploadPromises = [];
-    const uniquePageKey = getFileNameWithoutExt(pageDataZipFilePath);
+    const uniqueKey = getFileNameWithoutExt(dataZipFilePath);
     let dataMap = {
-      key: uniquePageKey,
+      key: uniqueKey,
       assets: {},
       data: ""
     };
 
-    _yauzl.default.open(pageDataZipFilePath, {
+    _yauzl.default.open(dataZipFilePath, {
       lazyEntries: true
     }, function (err, zipFile) {
       if (err) {
-        console.warn("ERROR: Failed to extract zip: ", pageDataZipFilePath, err);
+        console.warn("ERROR: Failed to extract zip: ", dataZipFilePath, err);
         reject(err);
+        return;
+      }
+
+      if (!zipFile) {
+        console.log("ERROR: Probably failed to extract zip: " + dataZipFilePath);
+        reject("Missing Zip File Resource.");
+        return;
       }
 
       console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
       zipFile.on("end", function (err) {
         if (err) {
-          console.warn('ERROR: Failed on "END" for file: ', pageDataZipFilePath, err);
+          console.warn('ERROR: Failed on "END" for file: ', dataZipFilePath, err);
           reject(err);
         }
 
@@ -509,15 +657,22 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
             if (err) {
               console.warn("ERROR: Failed while performing [openReadStream] for file: ", entry.fileName, err);
               reject(err);
+              return;
+            }
+
+            if (!readStream) {
+              console.log("ERROR: Missing Read Stream while importing.");
+              reject("Missing Read Strea Resource.");
+              return;
             }
 
             readStream.on("end", function () {
               filePaths.push(entry.fileName);
               zipFile.readEntry();
             });
-            const newKey = `${uniquePath}/${uniquePageKey}/${entry.fileName}`; // Modify in place
+            const newKey = `${uniquePath}/${uniqueKey}/${entry.fileName}`; // Modify in place
 
-            dataMap = preparePageDataDirMap({
+            dataMap = prepareDataDirMap({
               map: dataMap,
               filePath: entry.fileName,
               newKey
@@ -530,6 +685,8 @@ function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
 
             streamPipeline(readStream, streamPassThrough).then(() => {
               fileUploadPromises.push(promise);
+            }).catch(error => {
+              reject(error);
             });
           });
         }