@webiny/api-page-builder-import-export 0.0.0-mt-1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Webiny
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,17 @@
+# @webiny/api-page-builder-import-export
+[![](https://img.shields.io/npm/dw/@webiny/api-page-builder.svg)](https://www.npmjs.com/package/@webiny/api-page-builder) 
+[![](https://img.shields.io/npm/v/@webiny/api-page-builder.svg)](https://www.npmjs.com/package/@webiny/api-page-builder)
+[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)
+[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
+
+The API for the [Webiny Page Builder (@webiny/app-page-builder)](../app-page-builder) app. 
+  
+## Install
+```
+npm install --save @webiny/api-page-builder-import-export
+```
+
+Or if you prefer yarn: 
+```
+yarn add @webiny/api-page-builder-import-export
+```

package/exportPages/combine/index.d.ts

@@ -0,0 +1,19 @@
+import { HandlerPlugin } from "@webiny/handler/types";
+import { ArgsContext } from "@webiny/handler-args/types";
+import { PbPageImportExportContext } from "../../types";
+import { SecurityIdentity } from "@webiny/api-security/types";
+export declare type HandlerArgs = {
+    taskId: string;
+    identity: SecurityIdentity;
+};
+export declare type HandlerResponse = {
+    data: string;
+    error: {
+        message: string;
+    };
+};
+declare const _default: () => HandlerPlugin<PbPageImportExportContext, ArgsContext<HandlerArgs>>;
+/**
+ * Handles the export pages combine workflow.
+ */
+export default _default;

package/exportPages/combine/index.js

@@ -0,0 +1,88 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _types = require("../../types");
+
+var _s3Stream = require("../s3Stream");
+
+var _zipper = require("../zipper");
+
+var _mockSecurity = require("../../mockSecurity");
+
+/**
+ * Handles the export pages combine workflow.
+ */
+var _default = () => ({
+  type: "handler",
+
+  async handle(context) {
+    const log = console.log;
+    log("RUNNING Export Pages Combine Handler");
+    const {
+      invocationArgs: args,
+      pageBuilder
+    } = context;
+    const {
+      taskId,
+      identity
+    } = args;
+    (0, _mockSecurity.mockSecurity)(identity, context);
+
+    try {
+      const task = await pageBuilder.pageImportExportTask.getTask(taskId);
+      const {
+        exportPagesDataKey
+      } = task.input; // Get all files (zip) from given key
+
+      const listObjectResponse = await _s3Stream.s3Stream.listObject(exportPagesDataKey);
+      const zipFileKeys = listObjectResponse.Contents.filter(file => file.Key !== exportPagesDataKey).map(file => file.Key); // Prepare zip of all zips
+
+      const zipOfZip = new _zipper.ZipOfZip(zipFileKeys); // Upload
+
+      const pageExportUpload = await zipOfZip.process();
+      log(`Done uploading... File is located at ${pageExportUpload.Location} `); // Update task status and save export page data key
+
+      await pageBuilder.pageImportExportTask.updateTask(taskId, {
+        status: _types.PageImportExportTaskStatus.COMPLETED,
+        data: {
+          message: `Finish uploading page export.`,
+          key: pageExportUpload.Key,
+          url: pageExportUpload.Location
+        }
+      }); // Remove individual zip files from storage
+
+      const deleteFilePromises = zipFileKeys.map(key => _s3Stream.s3Stream.deleteObject(key));
+      await Promise.all(deleteFilePromises);
+      log(`Successfully deleted ${deleteFilePromises.length} zip files.`);
+    } catch (e) {
+      log("[EXPORT_PAGES_COMBINE] Error => ", e);
+      /**
+       * In case of error, we'll update the task status to "failed",
+       * so that, client can show notify the user appropriately.
+       */
+
+      await pageBuilder.pageImportExportTask.updateTask(taskId, {
+        status: _types.PageImportExportTaskStatus.FAILED,
+        error: {
+          name: e.name,
+          message: e.message,
+          stack: e.stack,
+          code: "IMPORT_FAILED"
+        }
+      });
+      return {
+        data: null,
+        error: {
+          message: e.message
+        }
+      };
+    }
+  }
+
+});
+
+exports.default = _default;

package/exportPages/process/index.d.ts

@@ -0,0 +1,26 @@
+import { HandlerPlugin } from "@webiny/handler/types";
+import { ArgsContext } from "@webiny/handler-args/types";
+import { PbPageImportExportContext } from "../../types";
+import { SecurityIdentity } from "@webiny/api-security/types";
+export declare type HandlerArgs = {
+    taskId: string;
+    subTaskIndex: number;
+    identity?: SecurityIdentity;
+};
+export declare type HandlerResponse = {
+    data: string;
+    error: {
+        message: string;
+    };
+};
+interface Configuration {
+    handlers: {
+        process: string;
+        combine: string;
+    };
+}
+declare const _default: (configuration: Configuration) => HandlerPlugin<PbPageImportExportContext, ArgsContext<HandlerArgs>>;
+/**
+ * Handles the export pages process workflow.
+ */
+export default _default;

package/exportPages/process/index.js

@@ -0,0 +1,204 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _types = require("../../types");
+
+var _utils = require("../../importPages/utils");
+
+var _client = require("../../importPages/client");
+
+var _handlerGraphql = require("@webiny/handler-graphql");
+
+var _utils2 = require("../utils");
+
+var _mockSecurity = require("../../mockSecurity");
+
+/**
+ * Handles the export pages process workflow.
+ */
+var _default = configuration => ({
+  type: "handler",
+
+  async handle(context) {
+    const log = console.log;
+    let subTask;
+    let noPendingTask = true;
+    let prevStatusOfSubTask = _types.PageImportExportTaskStatus.PENDING;
+    log("RUNNING Export Pages Process Handler");
+    const {
+      invocationArgs: args,
+      pageBuilder,
+      fileManager
+    } = context;
+    const {
+      taskId,
+      subTaskIndex,
+      identity
+    } = args; // Disable authorization; this is necessary because we call Page Builder CRUD methods which include authorization checks
+    // and this Lambda is invoked internally, without credentials.
+
+    (0, _mockSecurity.mockSecurity)(identity, context);
+
+    try {
+      /*
+       * Note: We're not going to DB for finding the next sub-task to process,
+       * because the data might be out of sync due to GSI eventual consistency.
+       */
+      subTask = await pageBuilder.pageImportExportTask.getSubTask(taskId, (0, _utils.zeroPad)(subTaskIndex));
+      /**
+       * Base condition!!
+       * Bail out early, if task not found or task's status is not "pending".
+       */
+
+      if (!subTask || subTask.status !== _types.PageImportExportTaskStatus.PENDING) {
+        noPendingTask = true;
+        return;
+      } else {
+        noPendingTask = false;
+      }
+
+      log(`Fetched sub task => ${subTask.id}`);
+      const {
+        input
+      } = subTask;
+      const {
+        pageId,
+        exportPagesDataKey,
+        revisionType
+      } = input;
+      /**
+       * At the moment, we only export a single revision of the page.
+       * It could be "published" or "latest" depending upon user input.
+       *
+       * Note: In case of no "published" revision available, we use the latest revision.
+       */
+
+      let page;
+
+      try {
+        if (revisionType === _types.PageExportRevisionType.PUBLISHED) {
+          // Get "published" page.
+          page = await pageBuilder.pages.getPublishedById({
+            id: pageId
+          });
+        } else {
+          // Get "latest" page.
+          page = await pageBuilder.pages.get(pageId);
+        }
+      } catch (e) {
+        // If we're looking for "published" page and doesn't found it, get latest page.
+        if (revisionType === _types.PageExportRevisionType.PUBLISHED && e instanceof _handlerGraphql.NotFoundError) {
+          page = await pageBuilder.pages.get(pageId);
+        } else {
+          throw e;
+        }
+      }
+
+      if (!page) {
+        log(`Unable to load page "${pageId}"`);
+        throw new _handlerGraphql.NotFoundError(`Unable to load page "${pageId}"`);
+      }
+
+      log(`Processing page key "${pageId}" | version ${page.version} | ${page.status}`); // Mark task status as PROCESSING
+
+      subTask = await pageBuilder.pageImportExportTask.updateSubTask(taskId, subTask.id, {
+        status: _types.PageImportExportTaskStatus.PROCESSING
+      }); // Update stats in main task
+
+      await pageBuilder.pageImportExportTask.updateStats(taskId, {
+        prevStatus: prevStatusOfSubTask,
+        nextStatus: _types.PageImportExportTaskStatus.PROCESSING
+      });
+      prevStatusOfSubTask = subTask.status;
+      log(`Extracting page data and uploading to storage...`); // Extract Page
+
+      const pageDataZip = await (0, _utils2.exportPage)(page, exportPagesDataKey, fileManager);
+      log(`Finish uploading zip...`); // Update task record in DB
+
+      subTask = await pageBuilder.pageImportExportTask.updateSubTask(taskId, subTask.id, {
+        status: _types.PageImportExportTaskStatus.COMPLETED,
+        data: {
+          message: `Finish uploading data for page "${page.id}" v${page.version} (${page.status}).`,
+          key: pageDataZip.Key
+        }
+      }); // Update stats in main task
+
+      await pageBuilder.pageImportExportTask.updateStats(taskId, {
+        prevStatus: prevStatusOfSubTask,
+        nextStatus: _types.PageImportExportTaskStatus.COMPLETED
+      });
+      prevStatusOfSubTask = subTask.status;
+    } catch (e) {
+      log("[EXPORT_PAGES_PROCESS] Error => ", e);
+
+      if (subTask && subTask.id) {
+        /**
+         * In case of error, we'll update the task status to "failed",
+         * so that, client can show notify the user appropriately.
+         */
+        const {
+          invocationArgs: args,
+          pageBuilder
+        } = context;
+        const {
+          taskId
+        } = args;
+        subTask = await pageBuilder.pageImportExportTask.updateSubTask(taskId, subTask.id, {
+          status: _types.PageImportExportTaskStatus.FAILED,
+          error: {
+            name: e.name,
+            message: e.message,
+            stack: e.stack,
+            code: "IMPORT_FAILED"
+          }
+        }); // Update stats in main task
+
+        await pageBuilder.pageImportExportTask.updateStats(taskId, {
+          prevStatus: prevStatusOfSubTask,
+          nextStatus: _types.PageImportExportTaskStatus.FAILED
+        });
+        prevStatusOfSubTask = subTask.status;
+      }
+
+      return {
+        data: null,
+        error: {
+          message: e.message
+        }
+      };
+    } finally {
+      // Base condition!
+      if (noPendingTask) {
+        log(`No pending sub-task for task ${taskId}`); // Combine individual page zip files.
+
+        await (0, _client.invokeHandlerClient)({
+          context,
+          name: configuration.handlers.combine,
+          payload: {
+            taskId,
+            identity: context.security.getIdentity()
+          }
+        });
+      } else {
+        console.log(`Invoking PROCESS for task "${subTaskIndex + 1}"`); // We want to continue with Self invocation no matter if current page error out.
+
+        await (0, _client.invokeHandlerClient)({
+          context,
+          name: configuration.handlers.process,
+          payload: {
+            taskId,
+            subTaskIndex: subTaskIndex + 1,
+            identity: context.security.getIdentity()
+          }
+        });
+      }
+    }
+  }
+
+});
+
+exports.default = _default;

package/exportPages/s3Stream.d.ts

@@ -0,0 +1,29 @@
+/// <reference types="node" />
+import { PassThrough, Readable } from "stream";
+import S3 from "aws-sdk/clients/s3";
+declare class S3Stream {
+    s3: S3;
+    bucket: string;
+    constructor();
+    /**
+     * We're checking if the file is accessible on S3 by getting object meta data.
+     * It help us to filter files that we need to download as part of export data.
+     * @param Key {string}
+     */
+    isFileAccessible(Key: string): Promise<boolean>;
+    getObjectHead(Key: string): Promise<S3.HeadObjectOutput>;
+    readStream(Key: string): Readable;
+    writeStream(Key: string, contentType?: string): {
+        streamPassThrough: PassThrough;
+        streamPassThroughUploadPromise: Promise<S3.ManagedUpload.SendData>;
+    };
+    upload(params: {
+        Key: string;
+        ContentType: string;
+        Body: Buffer;
+    }): Promise<S3.ManagedUpload.SendData>;
+    listObject(prefix: string): Promise<S3.ListObjectsOutput>;
+    deleteObject(key: string): Promise<S3.DeleteObjectOutput>;
+}
+export declare const s3Stream: S3Stream;
+export {};

package/exportPages/s3Stream.js

@@ -0,0 +1,106 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.s3Stream = void 0;
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+var _stream = require("stream");
+
+var _s = _interopRequireDefault(require("aws-sdk/clients/s3"));
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+
+const ARCHIVE_CONTENT_TYPE = "application/zip";
+
+class S3Stream {
+  constructor() {
+    (0, _defineProperty2.default)(this, "s3", void 0);
+    (0, _defineProperty2.default)(this, "bucket", void 0);
+    this.s3 = new _s.default({
+      region: process.env.AWS_REGION
+    });
+    this.bucket = process.env.S3_BUCKET;
+  }
+  /**
+   * We're checking if the file is accessible on S3 by getting object meta data.
+   * It help us to filter files that we need to download as part of export data.
+   * @param Key {string}
+   */
+
+
+  async isFileAccessible(Key) {
+    try {
+      await this.getObjectHead(Key);
+      return true;
+    } catch (error) {
+      console.warn(`Error while fetching meta data for file "${Key}"`);
+      console.log(error);
+      return false;
+    }
+  }
+
+  getObjectHead(Key) {
+    return this.s3.headObject({
+      Bucket: this.bucket,
+      Key
+    }).promise();
+  }
+
+  readStream(Key) {
+    return this.s3.getObject({
+      Bucket: this.bucket,
+      Key
+    }).createReadStream();
+  }
+
+  writeStream(Key, contentType = ARCHIVE_CONTENT_TYPE) {
+    const streamPassThrough = new _stream.Stream.PassThrough();
+    const params = {
+      ACL: "public-read",
+      Body: streamPassThrough,
+      Bucket: this.bucket,
+      ContentType: contentType,
+      Key
+    };
+    return {
+      streamPassThrough: streamPassThrough,
+
+      /**
+       * We're not using the `FileManager` storage plugin here because it currently doesn't support streams.
+       */
+      streamPassThroughUploadPromise: this.s3.upload(params).promise()
+    };
+  }
+
+  upload(params) {
+    return this.s3.upload(_objectSpread({
+      ACL: "private",
+      Bucket: this.bucket
+    }, params)).promise();
+  }
+
+  listObject(prefix) {
+    return this.s3.listObjects({
+      Bucket: this.bucket,
+      Prefix: prefix
+    }).promise();
+  }
+
+  deleteObject(key) {
+    return this.s3.deleteObject({
+      Key: key,
+      Bucket: this.bucket
+    }).promise();
+  }
+
+}
+
+const s3Stream = new S3Stream();
+exports.s3Stream = s3Stream;

package/exportPages/utils.d.ts

@@ -0,0 +1,13 @@
+import S3 from "aws-sdk/clients/s3";
+import { Page, File } from "@webiny/api-page-builder/types";
+import { FileManagerContext } from "@webiny/api-file-manager/types";
+export declare const EXPORT_PAGES_FOLDER_KEY = "WEBINY_PB_EXPORT_PAGES";
+export interface ExportedPageData {
+    page: Pick<Page, "content" | "title" | "version" | "status" | "settings" | "path">;
+    files: ImageFile[];
+}
+export declare function exportPage(page: Page, exportPagesDataKey: string, fileManager: FileManagerContext["fileManager"]): Promise<S3.ManagedUpload.SendData>;
+export interface ImageFile extends File {
+    key: string;
+}
+export declare function extractFilesFromPageData(data: Record<string, any>, files?: any[]): ImageFile[];

package/exportPages/utils.js

@@ -0,0 +1,113 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.EXPORT_PAGES_FOLDER_KEY = void 0;
+exports.exportPage = exportPage;
+exports.extractFilesFromPageData = extractFilesFromPageData;
+
+var _get = _interopRequireDefault(require("lodash/get"));
+
+var _s3Stream = require("./s3Stream");
+
+var _zipper = _interopRequireDefault(require("./zipper"));
+
+const EXPORT_PAGES_FOLDER_KEY = "WEBINY_PB_EXPORT_PAGES";
+exports.EXPORT_PAGES_FOLDER_KEY = EXPORT_PAGES_FOLDER_KEY;
+
+async function getFilteredFiles(files) {
+  const uniqueFileKeys = new Map();
+  const promises = files.map(file => _s3Stream.s3Stream.isFileAccessible(file.key));
+  const isFileAvailableResults = await Promise.all(promises);
+  const filesAvailableForDownload = []; // Filter files
+
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i]; // Check file accessibility
+
+    if (isFileAvailableResults[i] && !uniqueFileKeys.has(file.key)) {
+      filesAvailableForDownload.push(file);
+      uniqueFileKeys.set(file.key, true);
+    }
+  }
+
+  return filesAvailableForDownload;
+}
+
+async function exportPage(page, exportPagesDataKey, fileManager) {
+  // Extract all files
+  const files = extractFilesFromPageData(page.content); // Filter files
+
+  const filesAvailableForDownload = await getFilteredFiles(files); // Extract images from page settings
+
+  const pageSettingsImages = [(0, _get.default)(page, "settings.general.image"), (0, _get.default)(page, "settings.social.image")].filter(image => image && image.src);
+  const pageSettingsImagesData = []; // Get file data for all images inside "page.settings"
+
+  for (let i = 0; i < pageSettingsImages.length; i++) {
+    const {
+      id
+    } = pageSettingsImages[i];
+    const file = await fileManager.files.getFile(id);
+    pageSettingsImagesData.push(file);
+  } // Extract the page data in a json file and upload it to S3
+
+
+  const pageData = {
+    page: {
+      content: page.content,
+      title: page.title,
+      path: page.path,
+      version: page.version,
+      status: page.status,
+      settings: page.settings
+    },
+    files: [...filesAvailableForDownload, ...pageSettingsImagesData]
+  };
+  const pageDataBuffer = Buffer.from(JSON.stringify(pageData));
+  const zipper = new _zipper.default({
+    exportInfo: {
+      files: [...filesAvailableForDownload, ...pageSettingsImagesData],
+      pageTitle: page.title,
+      pageDataBuffer
+    },
+    archiveFileKey: exportPagesDataKey
+  });
+  return zipper.process();
+}
+
+function extractFilesFromPageData(data, files = []) {
+  // Base case: termination
+  if (!data || typeof data !== "object") {
+    return files;
+  } // Recursively call function for each element
+
+
+  if (Array.isArray(data)) {
+    for (let i = 0; i < data.length; i++) {
+      const element = data[i];
+      extractFilesFromPageData(element, files);
+    }
+
+    return files;
+  } // Main
+
+
+  const tuple = Object.entries(data);
+
+  for (let i = 0; i < tuple.length; i++) {
+    const [key, value] = tuple[i]; // TODO: @ashutosh extract it to plugins, so that, we can handle cases for other components too.
+
+    if (key === "file" && value) {
+      files.push(value);
+    } else if (key === "images" && Array.isArray(value)) {
+      // Handle case for "images-list" component
+      files.push(...value);
+    } else {
+      extractFilesFromPageData(value, files);
+    }
+  }
+
+  return files;
+}

package/exportPages/zipper.d.ts

@@ -0,0 +1,35 @@
+/// <reference types="node" />
+import S3 from "aws-sdk/clients/s3";
+import { Readable } from "stream";
+import { ImageFile } from "./utils";
+interface FileStreamDetails {
+    stream: Readable;
+    filename: string;
+}
+interface ExportInfo {
+    files: ImageFile[];
+    pageTitle: string;
+    pageDataBuffer: Buffer;
+}
+export interface ZipperConfig {
+    exportInfo: ExportInfo;
+    archiveFileKey: string;
+}
+export default class Zipper {
+    private readonly archiveFormat;
+    private readonly filesDirName;
+    private readonly archiveFileName;
+    config: ZipperConfig;
+    constructor(config: ZipperConfig);
+    s3DownloadStreams(): FileStreamDetails[];
+    process(): Promise<S3.ManagedUpload.SendData>;
+}
+export declare class ZipOfZip {
+    private readonly archiveFormat;
+    private readonly archiveFileName;
+    keys: string[];
+    constructor(keys: string[]);
+    getFileStreams(): FileStreamDetails[];
+    process(): Promise<S3.ManagedUpload.SendData>;
+}
+export {};