@webiny/api-page-builder-import-export 0.0.0-mt-1

package/importPages/utils.js

@@ -0,0 +1,539 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.importPage = importPage;
+exports.initialStats = initialStats;
+exports.readExtractAndUploadZipFileContents = readExtractAndUploadZipFileContents;
+exports.zeroPad = exports.uploadPageAssets = void 0;
+
+var _uniqid = _interopRequireDefault(require("uniqid"));
+
+var _dotPropImmutable = _interopRequireDefault(require("dot-prop-immutable"));
+
+var _fs = require("fs");
+
+var _fsExtra = require("fs-extra");
+
+var _util = require("util");
+
+var _stream = require("stream");
+
+var _nodeFetch = _interopRequireDefault(require("node-fetch"));
+
+var _path = _interopRequireDefault(require("path"));
+
+var _yauzl = _interopRequireDefault(require("yauzl"));
+
+var _chunk = _interopRequireDefault(require("lodash/chunk"));
+
+var _loadJsonFile = _interopRequireDefault(require("load-json-file"));
+
+var _error = _interopRequireDefault(require("@webiny/error"));
+
+var _downloadInstallFiles = require("@webiny/api-page-builder/graphql/crud/install/utils/downloadInstallFiles");
+
+var _types = require("../types");
+
+var _s3Stream = require("../exportPages/s3Stream");
+
+const streamPipeline = (0, _util.promisify)(_stream.pipeline);
+const INSTALL_DIR = "/tmp";
+
+const INSTALL_EXTRACT_DIR = _path.default.join(INSTALL_DIR, "apiPageBuilderImportPage");
+
+const FILES_COUNT_IN_EACH_BATCH = 15;
+const ZIP_CONTENT_TYPE = "application/zip";
+
+function updateImageInPageSettings({
+  settings,
+  fileIdToKeyMap,
+  srcPrefix
+}) {
+  let newSettings = settings;
+  const srcPrefixWithoutTrailingSlash = srcPrefix.endsWith("/") ? srcPrefix.slice(0, -1) : srcPrefix;
+
+  if (_dotPropImmutable.default.get(newSettings, "general.image.src")) {
+    newSettings = _dotPropImmutable.default.set(newSettings, "general.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(settings.general.image.id)}`);
+  }
+
+  if (_dotPropImmutable.default.get(newSettings, "social.image.src")) {
+    newSettings = _dotPropImmutable.default.set(newSettings, "social.image.src", `${srcPrefixWithoutTrailingSlash}/${fileIdToKeyMap.get(settings.social.image.id)}`);
+  }
+
+  return settings;
+}
+
+function updateFilesInPageData({
+  data,
+  fileIdToKeyMap,
+  srcPrefix
+}) {
+  // BASE CASE: Termination point
+  if (!data || typeof data !== "object") {
+    return;
+  } // Recursively call function if data is array
+
+
+  if (Array.isArray(data)) {
+    for (let i = 0; i < data.length; i++) {
+      const element = data[i];
+      updateFilesInPageData({
+        data: element,
+        fileIdToKeyMap,
+        srcPrefix
+      });
+    }
+
+    return;
+  } // Main logic
+
+
+  const tuple = Object.entries(data);
+
+  for (let i = 0; i < tuple.length; i++) {
+    const [key, value] = tuple[i];
+
+    if (key === "file" && value && fileIdToKeyMap.has(value.id)) {
+      value.key = fileIdToKeyMap.get(value.id);
+      value.name = fileIdToKeyMap.get(value.id);
+      value.src = `${srcPrefix}${srcPrefix.endsWith("/") ? "" : "/"}${fileIdToKeyMap.get(value.id)}`;
+    } else {
+      updateFilesInPageData({
+        data: value,
+        srcPrefix,
+        fileIdToKeyMap
+      });
+    }
+  }
+}
+
+const uploadPageAssets = async ({
+  context,
+  filesData,
+  fileUploadsData
+}) => {
+  /**
+   * This function contains logic of file download from S3.
+   * Current we're not mocking zip file download from S3 in tests at the moment.
+   * So, we're manually mocking it in case of test just by returning an empty object.
+   */
+  if (process.env.NODE_ENV === "test") {
+    return {};
+  }
+
+  console.log("INSIDE uploadPageAssets"); // Save uploaded file key against static id for later use.
+
+  const fileIdToKeyMap = new Map(); // Save files meta data against old key for later use.
+
+  const fileKeyToFileMap = new Map(); // Initialize maps.
+
+  for (let i = 0; i < filesData.length; i++) {
+    const file = filesData[i];
+    fileKeyToFileMap.set(file.key, file); // Initialize the value
+
+    fileIdToKeyMap.set(file.id, file.type);
+  }
+
+  const fileUploadResults = await uploadFilesFromS3({
+    fileKeyToFileMap,
+    oldKeyToNewKeyMap: fileUploadsData.assets
+  }); // Create files in File Manager
+
+  const createFilesInput = fileUploadResults.map(uploadResult => {
+    const newKey = uploadResult.Key;
+    const file = fileKeyToFileMap.get(getOldFileKey(newKey)); // Update the file map with newly uploaded file.
+
+    fileIdToKeyMap.set(file.id, newKey);
+    return {
+      key: newKey,
+      name: file.name,
+      size: file.size,
+      type: file.type,
+      meta: file.meta,
+      tags: file.tags
+    };
+  });
+  const createFilesPromises = []; // Gives an array of chunks (each consists of FILES_COUNT_IN_EACH_BATCH items).
+
+  const createFilesInputChunks = (0, _chunk.default)(createFilesInput, FILES_COUNT_IN_EACH_BATCH);
+
+  for (let i = 0; i < createFilesInputChunks.length; i++) {
+    const createFilesInputChunk = createFilesInputChunks[i];
+    createFilesPromises.push(
+    /*
+     * We need to break down files into chunks because
+     * `createFilesInBatch` operation has a limit on number of files it can handle at once.
+     */
+    context.fileManager.files.createFilesInBatch(createFilesInputChunk));
+  }
+
+  await Promise.all(createFilesPromises);
+  return {
+    fileIdToKeyMap
+  };
+};
+
+exports.uploadPageAssets = uploadPageAssets;
+
+async function importPage({
+  pageKey,
+  context,
+  fileUploadsData
+}) {
+  const log = console.log; // Making Directory for page in which we're going to extract the page data file.
+
+  const PAGE_EXTRACT_DIR = _path.default.join(INSTALL_EXTRACT_DIR, pageKey);
+
+  (0, _fsExtra.ensureDirSync)(PAGE_EXTRACT_DIR);
+
+  const pageDataFileKey = _dotPropImmutable.default.get(fileUploadsData, `data`);
+
+  const PAGE_DATA_FILE_PATH = _path.default.join(PAGE_EXTRACT_DIR, _path.default.basename(pageDataFileKey));
+
+  log(`Downloading Page data file: ${pageDataFileKey} at "${PAGE_DATA_FILE_PATH}"`); // Download and save page data file in disk.
+
+  await new Promise((resolve, reject) => {
+    _s3Stream.s3Stream.readStream(pageDataFileKey).on("error", reject).pipe((0, _fs.createWriteStream)(PAGE_DATA_FILE_PATH)).on("error", reject).on("finish", resolve);
+  }); // Load the page data file from disk.
+
+  log(`Load file ${pageDataFileKey}`);
+  const {
+    page,
+    files
+  } = await (0, _loadJsonFile.default)(PAGE_DATA_FILE_PATH); // Only update page data if there are files.
+
+  if (Array.isArray(files) && files.length) {
+    // Upload page assets.
+    const {
+      fileIdToKeyMap
+    } = await uploadPageAssets({
+      context,
+      filesData: files,
+      fileUploadsData
+    });
+    const {
+      srcPrefix
+    } = await context.fileManager.settings.getSettings();
+    updateFilesInPageData({
+      data: page.content,
+      fileIdToKeyMap,
+      srcPrefix
+    });
+    page.settings = updateImageInPageSettings({
+      settings: page.settings,
+      fileIdToKeyMap,
+      srcPrefix
+    });
+  }
+
+  log("Removing Directory for page...");
+  await (0, _downloadInstallFiles.deleteFile)(pageKey);
+  log(`Remove page contents from S3...`);
+  await deleteS3Folder(_path.default.dirname(fileUploadsData.data));
+  return page;
+}
+
+async function uploadFilesFromS3({
+  fileKeyToFileMap,
+  oldKeyToNewKeyMap
+}) {
+  const oldKeysForAssets = Object.keys(oldKeyToNewKeyMap);
+  const promises = []; // Upload all assets.
+
+  for (let i = 0; i < oldKeysForAssets.length; i++) {
+    const oldKey = oldKeysForAssets[i];
+    const tempNewKey = oldKeyToNewKeyMap[oldKey]; // Read file.
+
+    const readStream = _s3Stream.s3Stream.readStream(tempNewKey); // Get file meta data.
+
+
+    const fileMetaData = fileKeyToFileMap.get(oldKey);
+
+    if (fileMetaData) {
+      const newKey = (0, _uniqid.default)("", `-${fileMetaData.key}`);
+
+      const {
+        streamPassThrough,
+        streamPassThroughUploadPromise: promise
+      } = _s3Stream.s3Stream.writeStream(newKey, fileMetaData.type);
+
+      readStream.pipe(streamPassThrough);
+      promises.push(promise);
+      console.log(`Successfully queued file "${newKey}"`);
+    }
+  }
+
+  return Promise.all(promises);
+}
+
+async function getObjectMetaFromS3(Key) {
+  const meta = await _s3Stream.s3Stream.getObjectHead(Key);
+
+  if (meta.ContentType !== ZIP_CONTENT_TYPE) {
+    throw new _error.default(`Unsupported file type: "${meta.ContentType}"`, "UNSUPPORTED_FILE");
+  }
+}
+
+function getOldFileKey(key) {
+  /*
+   * Because we know the naming convention, we can extract the old key from new key.
+   */
+  try {
+    const [, ...rest] = key.split("-");
+    return rest.join("-");
+  } catch (e) {
+    return key;
+  }
+}
+
+const FILE_CONTENT_TYPE = "application/octet-stream";
+
+function getFileNameWithoutExt(fileName) {
+  return _path.default.basename(fileName).replace(_path.default.extname(fileName), "");
+}
+
+/**
+ * Function will read the given zip file from S3 via stream, extract its content and upload it to S3 bucket.
+ * @param zipFileKey
+ * @return PageImportData S3 file keys for all uploaded assets group by page.
+ */
+async function readExtractAndUploadZipFileContents(zipFileKey) {
+  const log = console.log;
+  const pageImportDataList = [];
+  let readStream; // Check whether it is a URL
+
+  if (zipFileKey.startsWith("http")) {
+    const response = await (0, _nodeFetch.default)(zipFileKey);
+
+    if (!response.ok) {
+      throw new _error.default(`Unable to downloading file: "${zipFileKey}"`, response.statusText);
+    }
+
+    readStream = response.body;
+  } else {
+    // We're first retrieving object's meta data, just to check whether the file is available at the given Key
+    await getObjectMetaFromS3(zipFileKey);
+    readStream = _s3Stream.s3Stream.readStream(zipFileKey);
+  }
+
+  const uniquePath = (0, _uniqid.default)("IMPORT_PAGES/");
+
+  const zipFileName = _path.default.basename(zipFileKey); // Read export file and download it in the disk
+
+
+  const ZIP_FILE_PATH = _path.default.join(INSTALL_DIR, zipFileName);
+
+  const writeStream = (0, _fs.createWriteStream)(ZIP_FILE_PATH);
+  await streamPipeline(readStream, writeStream);
+  log(`Downloaded file "${zipFileName}" at ${ZIP_FILE_PATH}`); // Extract the downloaded zip file
+
+  const zipFilePaths = await extractZipToDisk(ZIP_FILE_PATH);
+  log(`Removing ZIP file "${zipFileKey}" from ${ZIP_FILE_PATH}`);
+  await (0, _downloadInstallFiles.deleteFile)(ZIP_FILE_PATH); // Extract each page zip and upload their content's to S3
+
+  for (let i = 0; i < zipFilePaths.length; i++) {
+    const currentPath = zipFilePaths[i];
+    const dataMap = await extractZipAndUploadToS3(currentPath, uniquePath);
+    pageImportDataList.push(dataMap);
+  }
+
+  log("Removing all ZIP files located at ", _path.default.dirname(zipFilePaths[0]));
+  await (0, _downloadInstallFiles.deleteFile)(_path.default.dirname(zipFilePaths[0]));
+  return pageImportDataList;
+}
+
+const ASSETS_DIR_NAME = "/assets";
+
+function preparePageDataDirMap({
+  map,
+  filePath,
+  newKey
+}) {
+  const dirname = _path.default.dirname(filePath);
+
+  const fileName = _path.default.basename(filePath);
+  /*
+   * We want to use dot (.) as part of object key rather than creating nested object(s).
+   * Also, the file name might contain dots in it beside the extension, so, we are escaping them all.
+   */
+
+
+  const oldKey = fileName.replace(/\./g, "\\.");
+  const isAsset = dirname.endsWith(ASSETS_DIR_NAME);
+
+  if (isAsset) {
+    map = _dotPropImmutable.default.set(map, `assets.${oldKey}`, newKey);
+  } else {
+    // We only need to know the newKey for data file.
+    map = _dotPropImmutable.default.set(map, `data`, newKey);
+  }
+
+  return map;
+}
+
+async function deleteS3Folder(key) {
+  // Append trailing slash i.e "/" to key to make sure we only delete a specific folder.
+  if (!key.endsWith("/")) {
+    key = `${key}/`;
+  }
+
+  const response = await _s3Stream.s3Stream.listObject(key);
+  const keys = response.Contents.map(c => c.Key);
+  console.log(`Found ${keys.length} files.`);
+  const deleteFilePromises = keys.map(key => _s3Stream.s3Stream.deleteObject(key));
+  await Promise.all(deleteFilePromises);
+  console.log(`Successfully deleted ${deleteFilePromises.length} files.`);
+}
+
+const zeroPad = version => `${version}`.padStart(5, "0");
+
+exports.zeroPad = zeroPad;
+
+function initialStats(total) {
+  return {
+    [_types.PageImportExportTaskStatus.PENDING]: total,
+    [_types.PageImportExportTaskStatus.PROCESSING]: 0,
+    [_types.PageImportExportTaskStatus.COMPLETED]: 0,
+    [_types.PageImportExportTaskStatus.FAILED]: 0,
+    total
+  };
+}
+
+function extractZipToDisk(exportFileZipPath) {
+  return new Promise((resolve, reject) => {
+    const pageZipFilePaths = [];
+    const uniqueFolderNameForExport = getFileNameWithoutExt(exportFileZipPath);
+
+    const EXPORT_FILE_EXTRACTION_PATH = _path.default.join(INSTALL_DIR, uniqueFolderNameForExport); // Make sure DIR exists
+
+
+    (0, _fsExtra.ensureDirSync)(EXPORT_FILE_EXTRACTION_PATH);
+
+    _yauzl.default.open(exportFileZipPath, {
+      lazyEntries: true
+    }, function (err, zipFile) {
+      if (err) {
+        console.warn("ERROR: Failed to extract zip: ", exportFileZipPath, err);
+        reject(err);
+      }
+
+      console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
+      zipFile.on("end", function (err) {
+        if (err) {
+          console.warn("ERROR: Failed on END event for file: ", exportFileZipPath, err);
+          reject(err);
+        }
+
+        resolve(pageZipFilePaths);
+      });
+      zipFile.readEntry();
+      zipFile.on("entry", function (entry) {
+        console.info(`Processing entry: "${entry.fileName}"`);
+
+        if (/\/$/.test(entry.fileName)) {
+          // Directory file names end with '/'.
+          // Note that entries for directories themselves are optional.
+          // An entry's fileName implicitly requires its parent directories to exist.
+          zipFile.readEntry();
+        } else {
+          // file entry
+          zipFile.openReadStream(entry, function (err, readStream) {
+            if (err) {
+              console.warn("ERROR: Failed to openReadStream for file: ", entry.fileName, err);
+              reject(err);
+            }
+
+            const filePath = _path.default.join(EXPORT_FILE_EXTRACTION_PATH, entry.fileName);
+
+            readStream.on("end", function () {
+              pageZipFilePaths.push(filePath);
+              zipFile.readEntry();
+            });
+            streamPipeline(readStream, (0, _fs.createWriteStream)(filePath));
+          });
+        }
+      });
+    });
+  });
+}
+
+function extractZipAndUploadToS3(pageDataZipFilePath, uniquePath) {
+  return new Promise((resolve, reject) => {
+    const filePaths = [];
+    const fileUploadPromises = [];
+    const uniquePageKey = getFileNameWithoutExt(pageDataZipFilePath);
+    let dataMap = {
+      key: uniquePageKey,
+      assets: {},
+      data: ""
+    };
+
+    _yauzl.default.open(pageDataZipFilePath, {
+      lazyEntries: true
+    }, function (err, zipFile) {
+      if (err) {
+        console.warn("ERROR: Failed to extract zip: ", pageDataZipFilePath, err);
+        reject(err);
+      }
+
+      console.info(`The ZIP file contains ${zipFile.entryCount} entries.`);
+      zipFile.on("end", function (err) {
+        if (err) {
+          console.warn('ERROR: Failed on "END" for file: ', pageDataZipFilePath, err);
+          reject(err);
+        }
+
+        Promise.all(fileUploadPromises).then(res => {
+          res.forEach(r => {
+            console.info("Done uploading... ", r);
+          });
+          resolve(dataMap);
+        });
+      });
+      zipFile.readEntry();
+      zipFile.on("entry", function (entry) {
+        console.info(`Processing entry: "${entry.fileName}"`);
+
+        if (/\/$/.test(entry.fileName)) {
+          // Directory file names end with '/'.
+          // Note that entries for directories themselves are optional.
+          // An entry's fileName implicitly requires its parent directories to exist.
+          zipFile.readEntry();
+        } else {
+          // file entry
+          zipFile.openReadStream(entry, function (err, readStream) {
+            if (err) {
+              console.warn("ERROR: Failed while performing [openReadStream] for file: ", entry.fileName, err);
+              reject(err);
+            }
+
+            readStream.on("end", function () {
+              filePaths.push(entry.fileName);
+              zipFile.readEntry();
+            });
+            const newKey = `${uniquePath}/${uniquePageKey}/${entry.fileName}`; // Modify in place
+
+            dataMap = preparePageDataDirMap({
+              map: dataMap,
+              filePath: entry.fileName,
+              newKey
+            });
+
+            const {
+              streamPassThrough,
+              streamPassThroughUploadPromise: promise
+            } = _s3Stream.s3Stream.writeStream(newKey, FILE_CONTENT_TYPE);
+
+            streamPipeline(readStream, streamPassThrough).then(() => {
+              fileUploadPromises.push(promise);
+            });
+          });
+        }
+      });
+    });
+  });
+}

package/mockSecurity.d.ts

@@ -0,0 +1,2 @@
+import { SecurityContext, SecurityIdentity } from "@webiny/api-security/types";
+export declare const mockSecurity: (identity: SecurityIdentity, context: SecurityContext) => void;

package/mockSecurity.js

@@ -0,0 +1,13 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.mockSecurity = void 0;
+
+const mockSecurity = (identity, context) => {
+  context.security.disableAuthorization();
+  context.security.setIdentity(identity);
+};
+
+exports.mockSecurity = mockSecurity;

package/package.json

@@ -0,0 +1,80 @@
+{
+	"name": "@webiny/api-page-builder-import-export",
+	"version": "0.0.0-mt-1",
+	"main": "index.js",
+	"keywords": [
+		"pbie:base"
+	],
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/webiny/webiny-js.git",
+		"directory": "packages/api-page-builder-import-export"
+	},
+	"description": "The API for the import export feature of the Webiny Page Builder app.",
+	"author": "Webiny Ltd",
+	"license": "MIT",
+	"dependencies": {
+		"@babel/runtime": "7.15.4",
+		"@commodo/fields": "1.1.2-beta.20",
+		"@webiny/api-file-manager": "0.0.0-mt-1",
+		"@webiny/api-page-builder": "0.0.0-mt-1",
+		"@webiny/api-security": "0.0.0-mt-1",
+		"@webiny/error": "0.0.0-mt-1",
+		"@webiny/handler": "0.0.0-mt-1",
+		"@webiny/handler-args": "0.0.0-mt-1",
+		"@webiny/handler-aws": "0.0.0-mt-1",
+		"@webiny/handler-graphql": "0.0.0-mt-1",
+		"@webiny/validation": "0.0.0-mt-1",
+		"archiver": "5.3.0",
+		"commodo-fields-object": "1.0.6",
+		"dot-prop-immutable": "2.1.1",
+		"fs-extra": "7.0.1",
+		"load-json-file": "6.2.0",
+		"lodash": "4.17.21",
+		"mdbid": "1.0.0",
+		"node-fetch": "2.6.5",
+		"stream": "0.0.2",
+		"uniqid": "5.4.0",
+		"yauzl": "2.10.0"
+	},
+	"devDependencies": {
+		"@babel/cli": "^7.5.5",
+		"@babel/core": "^7.5.5",
+		"@babel/plugin-proposal-export-default-from": "^7.5.2",
+		"@babel/preset-env": "^7.5.5",
+		"@babel/preset-typescript": "^7.8.3",
+		"@elastic/elasticsearch": "7.12.0",
+		"@shelf/jest-elasticsearch": "^1.0.0",
+		"@types/puppeteer": "^5.4.2",
+		"@webiny/api-dynamodb-to-elasticsearch": "^0.0.0-mt-1",
+		"@webiny/api-file-manager-ddb-es": "^0.0.0-mt-1",
+		"@webiny/api-i18n-ddb": "^0.0.0-mt-1",
+		"@webiny/api-security-so-ddb": "^0.0.0-mt-1",
+		"@webiny/api-tenancy": "^0.0.0-mt-1",
+		"@webiny/api-tenancy-so-ddb": "^0.0.0-mt-1",
+		"@webiny/cli": "^0.0.0-mt-1",
+		"@webiny/db": "^0.0.0-mt-1",
+		"@webiny/project-utils": "^0.0.0-mt-1",
+		"jest": "^26.6.3",
+		"jest-dynalite": "^3.2.0",
+		"rimraf": "^3.0.2",
+		"ttypescript": "^1.5.12",
+		"typescript": "^4.1.3"
+	},
+	"publishConfig": {
+		"access": "public",
+		"directory": "dist"
+	},
+	"scripts": {
+		"build": "yarn webiny run build",
+		"watch": "yarn webiny run watch"
+	},
+	"adio": {
+		"ignore": {
+			"src": [
+				"aws-sdk"
+			]
+		}
+	},
+	"gitHead": "37736d8456a6ecb342a6c3645060bd9a3f2d4bb0"
+}