@webiny/api-page-builder-import-export 0.0.0-unstable.990c3ab1b6 → 0.0.0-unstable.9bd236cf5e

package/utils/ZipFiles.js

@@ -0,0 +1,124 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ZipFiles = void 0;
+var _archiver = require("archiver");
+var _clientS = require("@webiny/aws-sdk/client-s3");
+var _path = _interopRequireDefault(require("path"));
+var _stream = require("stream");
+var _libStorage = require("@webiny/aws-sdk/lib-storage");
+var _nodeHttpHandler = require("@smithy/node-http-handler");
+var _https = require("https");
+var _http = require("http");
+class ZipFiles {
+  bucket = process.env.S3_BUCKET;
+  debug = process.env.DEBUG === "true";
+  constructor(options) {
+    this.setDebug(options?.debug);
+  }
+  async process(targetFileName, files) {
+    const fileNames = Array.from(files);
+    const s3Client = (0, _clientS.createS3)({
+      requestHandler: new _nodeHttpHandler.NodeHttpHandler({
+        connectionTimeout: 0,
+        httpAgent: new _http.Agent({
+          maxSockets: 10000,
+          keepAlive: true,
+          maxFreeSockets: 10000,
+          maxTotalSockets: 10000,
+          keepAliveMsecs: 900000 // milliseconds / 15 minutes
+        }),
+        httpsAgent: new _https.Agent({
+          maxSockets: 10000,
+          keepAlive: true,
+          sessionTimeout: 900,
+          // seconds / 15 minutes
+          maxCachedSessions: 100000,
+          maxFreeSockets: 10000,
+          maxTotalSockets: 10000,
+          keepAliveMsecs: 900000 // milliseconds / 15 minutes
+        }),
+        requestTimeout: 900000 // milliseconds / 15 minutes
+      })
+    });
+    const streamPassThrough = new _stream.Stream.PassThrough({
+      autoDestroy: true
+    });
+    const upload = new _libStorage.Upload({
+      client: s3Client,
+      params: {
+        ACL: "private",
+        Body: streamPassThrough,
+        Bucket: this.bucket,
+        ContentType: "application/zip",
+        Key: targetFileName
+      },
+      queueSize: 1,
+      partSize: 1024 * 1024 * 5,
+      leavePartsOnError: false
+    });
+    const archive = (0, _archiver.create)("zip", {});
+    archive.on("error", error => {
+      console.error(error);
+      throw new Error(`${error.name} ${error.code} ${error.message} ${error.path} ${error.stack}`);
+    });
+    archive.pipe(streamPassThrough);
+
+    /**
+     * To combine all the files into a single zip file, we need to add files one by one.
+     *
+     * addFileToArchive() method is called every time an entry event is triggered on the archive - it means that file was added into the archive.
+     * The method is called manually, first time, to start the process.
+     */
+
+    archive.on("entry", data => {
+      this.debug && console.log(`Archived file: ${data.name}`);
+      addFileToArchive();
+    });
+    const addFileToArchive = async () => {
+      const file = fileNames.shift();
+      if (!file) {
+        this.debug && console.log("No more files to add to the archive.");
+        /**
+         * Must call finalize() with a timeout, otherwise the lambda crashes.
+         */
+        setTimeout(() => {
+          archive.finalize();
+        }, 200);
+        return;
+      }
+      this.debug && console.log(`Adding file "${file}" to the archive.`);
+      const cmd = new _clientS.GetObjectCommand({
+        Bucket: this.bucket,
+        Key: file
+      });
+      const response = await s3Client.send(cmd);
+      // Possible to get a null response.Body?
+      // Typescript says yes, so let's check it.
+      if (!response.Body) {
+        this.debug && console.log(`No response.Body for file "${file}", moving to next file.`);
+        return addFileToArchive();
+      }
+      const name = `${_path.default.basename(file)}`;
+      archive.append(response.Body, {
+        name
+      });
+    };
+    addFileToArchive();
+    const result = await upload.done();
+    s3Client.destroy();
+    return result;
+  }
+  setDebug(debug) {
+    if (debug === undefined) {
+      return;
+    }
+    this.debug = debug;
+  }
+}
+exports.ZipFiles = ZipFiles;
+
+//# sourceMappingURL=ZipFiles.js.map

package/utils/ZipFiles.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_archiver","require","_clientS","_path","_interopRequireDefault","_stream","_libStorage","_nodeHttpHandler","_https","_http","ZipFiles","bucket","process","env","S3_BUCKET","debug","DEBUG","constructor","options","setDebug","targetFileName","files","fileNames","Array","from","s3Client","createS3","requestHandler","NodeHttpHandler","connectionTimeout","httpAgent","HttpAgent","maxSockets","keepAlive","maxFreeSockets","maxTotalSockets","keepAliveMsecs","httpsAgent","HttpsAgent","sessionTimeout","maxCachedSessions","requestTimeout","streamPassThrough","Stream","PassThrough","autoDestroy","upload","Upload","client","params","ACL","Body","Bucket","ContentType","Key","queueSize","partSize","leavePartsOnError","archive","createArchiver","on","error","console","Error","name","code","message","path","stack","pipe","data","log","addFileToArchive","file","shift","setTimeout","finalize","cmd","GetObjectCommand","response","send","basename","append","result","done","destroy","undefined","exports"],"sources":["ZipFiles.ts"],"sourcesContent":["import type { ArchiverError } from \"archiver\";\nimport { create as createArchiver } from \"archiver\";\nimport type { CompleteMultipartUploadOutput } from \"@webiny/aws-sdk/client-s3\";\nimport { createS3, GetObjectCommand } from \"@webiny/aws-sdk/client-s3\";\nimport path from \"path\";\nimport { type Readable, Stream } from \"stream\";\nimport { Upload } from \"@webiny/aws-sdk/lib-storage\";\nimport { NodeHttpHandler } from \"@smithy/node-http-handler\";\nimport { Agent as HttpsAgent } from \"https\";\nimport { Agent as HttpAgent } from \"http\";\n\nexport interface ZipFilesOptions {\n    debug?: boolean;\n}\n\nexport class ZipFiles {\n    private readonly bucket: string = process.env.S3_BUCKET as string;\n    private debug: boolean = process.env.DEBUG === \"true\";\n\n    public constructor(options?: ZipFilesOptions) {\n        this.setDebug(options?.debug);\n    }\n\n    public async process(\n        targetFileName: string,\n        files: string[]\n    ): Promise<CompleteMultipartUploadOutput> {\n        const fileNames = Array.from(files);\n        const s3Client = createS3({\n            requestHandler: new NodeHttpHandler({\n                connectionTimeout: 0,\n                httpAgent: new HttpAgent({\n                    maxSockets: 10000,\n                    keepAlive: true,\n                    maxFreeSockets: 10000,\n                    maxTotalSockets: 10000,\n                    keepAliveMsecs: 900000 // milliseconds / 15 minutes\n                }),\n                httpsAgent: new HttpsAgent({\n                    maxSockets: 10000,\n                    keepAlive: true,\n                    sessionTimeout: 900, // seconds / 15 minutes\n                    maxCachedSessions: 100000,\n                    maxFreeSockets: 10000,\n                    maxTotalSockets: 10000,\n                    keepAliveMsecs: 900000 // milliseconds / 15 minutes\n                }),\n                requestTimeout: 900000 // milliseconds / 15 minutes\n            })\n        });\n\n        const streamPassThrough = new Stream.PassThrough({\n            autoDestroy: true\n        });\n\n        const upload = new Upload({\n            client: s3Client,\n            params: {\n                ACL: \"private\",\n                Body: streamPassThrough,\n                Bucket: this.bucket,\n                ContentType: \"application/zip\",\n                Key: targetFileName\n            },\n            queueSize: 1,\n            partSize: 1024 * 1024 * 5,\n            leavePartsOnError: false\n        });\n\n        const archive = createArchiver(\"zip\", {});\n\n        archive.on(\"error\", (error: ArchiverError) => {\n            console.error(error);\n            throw new Error(\n                `${error.name} ${error.code} ${error.message} ${error.path} ${error.stack}`\n            );\n        });\n\n        archive.pipe(streamPassThrough);\n\n        /**\n         * To combine all the files into a single zip file, we need to add files one by one.\n         *\n         * addFileToArchive() method is called every time an entry event is triggered on the archive - it means that file was added into the archive.\n         * The method is called manually, first time, to start the process.\n         */\n\n        archive.on(\"entry\", data => {\n            this.debug && console.log(`Archived file: ${data.name}`);\n            addFileToArchive();\n        });\n\n        const addFileToArchive = async (): Promise<void> => {\n            const file = fileNames.shift();\n            if (!file) {\n                this.debug && console.log(\"No more files to add to the archive.\");\n                /**\n                 * Must call finalize() with a timeout, otherwise the lambda crashes.\n                 */\n                setTimeout(() => {\n                    archive.finalize();\n                }, 200);\n                return;\n            }\n            this.debug && console.log(`Adding file \"${file}\" to the archive.`);\n            const cmd = new GetObjectCommand({\n                Bucket: this.bucket,\n                Key: file\n            });\n\n            const response = await s3Client.send(cmd);\n            // Possible to get a null response.Body?\n            // Typescript says yes, so let's check it.\n            if (!response.Body) {\n                this.debug &&\n                    console.log(`No response.Body for file \"${file}\", moving to next file.`);\n                return addFileToArchive();\n            }\n            const name = `${path.basename(file)}`;\n\n            archive.append(response.Body as Readable, {\n                name\n            });\n        };\n\n        addFileToArchive();\n\n        const result = await upload.done();\n\n        s3Client.destroy();\n        return result;\n    }\n\n    private setDebug(debug?: boolean): void {\n        if (debug === undefined) {\n            return;\n        }\n        this.debug = debug;\n    }\n}\n"],"mappings":";;;;;;;AACA,IAAAA,SAAA,GAAAC,OAAA;AAEA,IAAAC,QAAA,GAAAD,OAAA;AACA,IAAAE,KAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,OAAA,GAAAJ,OAAA;AACA,IAAAK,WAAA,GAAAL,OAAA;AACA,IAAAM,gBAAA,GAAAN,OAAA;AACA,IAAAO,MAAA,GAAAP,OAAA;AACA,IAAAQ,KAAA,GAAAR,OAAA;AAMO,MAAMS,QAAQ,CAAC;EACDC,MAAM,GAAWC,OAAO,CAACC,GAAG,CAACC,SAAS;EAC/CC,KAAK,GAAYH,OAAO,CAACC,GAAG,CAACG,KAAK,KAAK,MAAM;EAE9CC,WAAWA,CAACC,OAAyB,EAAE;IAC1C,IAAI,CAACC,QAAQ,CAACD,OAAO,EAAEH,KAAK,CAAC;EACjC;EAEA,MAAaH,OAAOA,CAChBQ,cAAsB,EACtBC,KAAe,EACuB;IACtC,MAAMC,SAAS,GAAGC,KAAK,CAACC,IAAI,CAACH,KAAK,CAAC;IACnC,MAAMI,QAAQ,GAAG,IAAAC,iBAAQ,EAAC;MACtBC,cAAc,EAAE,IAAIC,gCAAe,CAAC;QAChCC,iBAAiB,EAAE,CAAC;QACpBC,SAAS,EAAE,IAAIC,WAAS,CAAC;UACrBC,UAAU,EAAE,KAAK;UACjBC,SAAS,EAAE,IAAI;UACfC,cAAc,EAAE,KAAK;UACrBC,eAAe,EAAE,KAAK;UACtBC,cAAc,EAAE,MAAM,CAAC;QAC3B,CAAC,CAAC;QACFC,UAAU,EAAE,IAAIC,YAAU,CAAC;UACvBN,UAAU,EAAE,KAAK;UACjBC,SAAS,EAAE,IAAI;UACfM,cAAc,EAAE,GAAG;UAAE;UACrBC,iBAAiB,EAAE,MAAM;UACzBN,cAAc,EAAE,KAAK;UACrBC,eAAe,EAAE,KAAK;UACtBC,cAAc,EAAE,MAAM,CAAC;QAC3B,CAAC,CAAC;QACFK,cAAc,EAAE,MAAM,CAAC;MAC3B,CAAC;IACL,CAAC,CAAC;IAEF,MAAMC,iBAAiB,GAAG,IAAIC,cAAM,CAACC,WAAW,CAAC;MAC7CC,WAAW,EAAE;IACjB,CAAC,CAAC;IAEF,MAAMC,MAAM,GAAG,IAAIC,kBAAM,CAAC;MACtBC,MAAM,EAAEvB,QAAQ;MAChBwB,MAAM,EAAE;QACJC,GAAG,EAAE,SAAS;QACdC,IAAI,EAAET,iBAAiB;QACvBU,MAAM,EAAE,IAAI,CAACzC,MAAM;QACnB0C,WAAW,EAAE,iBAAiB;QAC9BC,GAAG,EAAElC;MACT,CAAC;MACDmC,SAAS,EAAE,CAAC;MACZC,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,CAAC;MACzBC,iBAAiB,EAAE;IACvB,CAAC,CAAC;IAEF,MAAMC,OAAO,GAAG,IAAAC,gBAAc,EAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAEzCD,OAAO,CAACE,EAAE,CAAC,OAAO,EAAGC,KAAoB,IAAK;MAC1CC,OAAO,CAACD,KAAK,CAACA,KAAK,CAAC;MACpB,MAAM,IAAIE,KAAK,CACX,GAAGF,KAAK,CAACG,IAAI,IAAIH,KAAK,CAACI,IAAI,IAAIJ,KAAK,CAACK,OAAO,IAAIL,KAAK,CAACM,IAAI,IAAIN,KAAK,CAACO,KAAK,EAC7E,CAAC;IACL,CAAC,CAAC;IAEFV,OAAO,CAACW,IAAI,CAAC3B,iBAAiB,CAAC;;IAE/B;AACR;AACA;AACA;AACA;AACA;;IAEQgB,OAAO,CAACE,EAAE,CAAC,OAAO,EAAEU,IAAI,IAAI;MACxB,IAAI,CAACvD,KAAK,IAAI+C,OAAO,CAACS,GAAG,CAAC,kBAAkBD,IAAI,CAACN,IAAI,EAAE,CAAC;MACxDQ,gBAAgB,CAAC,CAAC;IACtB,CAAC,CAAC;IAEF,MAAMA,gBAAgB,GAAG,MAAAA,CAAA,KAA2B;MAChD,MAAMC,IAAI,GAAGnD,SAAS,CAACoD,KAAK,CAAC,CAAC;MAC9B,IAAI,CAACD,IAAI,EAAE;QACP,IAAI,CAAC1D,KAAK,IAAI+C,OAAO,CAACS,GAAG,CAAC,sCAAsC,CAAC;QACjE;AAChB;AACA;QACgBI,UAAU,CAAC,MAAM;UACbjB,OAAO,CAACkB,QAAQ,CAAC,CAAC;QACtB,CAAC,EAAE,GAAG,CAAC;QACP;MACJ;MACA,IAAI,CAAC7D,KAAK,IAAI+C,OAAO,CAACS,GAAG,CAAC,gBAAgBE,IAAI,mBAAmB,CAAC;MAClE,MAAMI,GAAG,GAAG,IAAIC,yBAAgB,CAAC;QAC7B1B,MAAM,EAAE,IAAI,CAACzC,MAAM;QACnB2C,GAAG,EAAEmB;MACT,CAAC,CAAC;MAEF,MAAMM,QAAQ,GAAG,MAAMtD,QAAQ,CAACuD,IAAI,CAACH,GAAG,CAAC;MACzC;MACA;MACA,IAAI,CAACE,QAAQ,CAAC5B,IAAI,EAAE;QAChB,IAAI,CAACpC,KAAK,IACN+C,OAAO,CAACS,GAAG,CAAC,8BAA8BE,IAAI,yBAAyB,CAAC;QAC5E,OAAOD,gBAAgB,CAAC,CAAC;MAC7B;MACA,MAAMR,IAAI,GAAG,GAAGG,aAAI,CAACc,QAAQ,CAACR,IAAI,CAAC,EAAE;MAErCf,OAAO,CAACwB,MAAM,CAACH,QAAQ,CAAC5B,IAAI,EAAc;QACtCa;MACJ,CAAC,CAAC;IACN,CAAC;IAEDQ,gBAAgB,CAAC,CAAC;IAElB,MAAMW,MAAM,GAAG,MAAMrC,MAAM,CAACsC,IAAI,CAAC,CAAC;IAElC3D,QAAQ,CAAC4D,OAAO,CAAC,CAAC;IAClB,OAAOF,MAAM;EACjB;EAEQhE,QAAQA,CAACJ,KAAe,EAAQ;IACpC,IAAIA,KAAK,KAAKuE,SAAS,EAAE;MACrB;IACJ;IACA,IAAI,CAACvE,KAAK,GAAGA,KAAK;EACtB;AACJ;AAACwE,OAAA,CAAA7E,QAAA,GAAAA,QAAA","ignoreList":[]}

package/exportPages/combine/index.js

@@ -1,106 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-var _types = require("../../types");
-var _s3Stream = require("../s3Stream");
-var _zipper = require("../zipper");
-var _mockSecurity = require("../../mockSecurity");
-var _handlerAws = require("@webiny/handler-aws");
-/**
- * Handles the export pages combine workflow.
- */
-var _default = () => {
-  return (0, _handlerAws.createRawEventHandler)(async ({
-    payload,
-    context
-  }) => {
-    const log = console.log;
-    log("RUNNING Export Pages Combine Handler");
-    const {
-      pageBuilder
-    } = context;
-    const {
-      taskId,
-      identity
-    } = payload;
-    (0, _mockSecurity.mockSecurity)(identity, context);
-    try {
-      const task = await pageBuilder.pageImportExportTask.getTask(taskId);
-      if (!task) {
-        return {
-          data: null,
-          error: {
-            message: `There is no task with ID "${taskId}".`
-          }
-        };
-      }
-      const {
-        exportPagesDataKey
-      } = task.input;
-
-      // Get all files (zip) from given key
-      const listObjectResponse = await _s3Stream.s3Stream.listObject(exportPagesDataKey);
-      if (!listObjectResponse.Contents) {
-        return {
-          data: null,
-          error: {
-            message: "There is no Contents defined on S3 Stream while combining pages."
-          }
-        };
-      }
-      const zipFileKeys = listObjectResponse.Contents.filter(file => file.Key !== exportPagesDataKey).map(file => file.Key).filter(Boolean);
-
-      // Prepare zip of all zips
-      const zipOfZip = new _zipper.ZipOfZip(zipFileKeys);
-
-      // Upload
-      const pageExportUpload = await zipOfZip.process();
-      log(`Done uploading... File is located at ${pageExportUpload.Location} `);
-
-      // Update task status and save export page data key
-      await pageBuilder.pageImportExportTask.updateTask(taskId, {
-        status: _types.PageImportExportTaskStatus.COMPLETED,
-        data: {
-          message: `Finish uploading page export.`,
-          key: pageExportUpload.Key,
-          url: _s3Stream.s3Stream.getPresignedUrl(pageExportUpload.Key)
-        }
-      });
-
-      // Remove individual zip files from storage
-      const deleteFilePromises = zipFileKeys.map(key => _s3Stream.s3Stream.deleteObject(key));
-      await Promise.all(deleteFilePromises);
-      log(`Successfully deleted ${deleteFilePromises.length} zip files.`);
-    } catch (e) {
-      log("[EXPORT_PAGES_COMBINE] Error => ", e);
-
-      /**
-       * In case of error, we'll update the task status to "failed",
-       * so that, client can show notify the user appropriately.
-       */
-      await pageBuilder.pageImportExportTask.updateTask(taskId, {
-        status: _types.PageImportExportTaskStatus.FAILED,
-        error: {
-          name: e.name,
-          message: e.message,
-          stack: e.stack,
-          code: "EXPORT_FAILED"
-        }
-      });
-      return {
-        data: null,
-        error: {
-          message: e.message
-        }
-      };
-    }
-    return {
-      data: "",
-      error: null
-    };
-  });
-};
-exports.default = _default;

package/exportPages/combine/index.js.map

@@ -1 +0,0 @@
-{"version":3,"names":["createRawEventHandler","payload","context","log","console","pageBuilder","taskId","identity","mockSecurity","task","pageImportExportTask","getTask","data","error","message","exportPagesDataKey","input","listObjectResponse","s3Stream","listObject","Contents","zipFileKeys","filter","file","Key","map","Boolean","zipOfZip","ZipOfZip","pageExportUpload","process","Location","updateTask","status","PageImportExportTaskStatus","COMPLETED","key","url","getPresignedUrl","deleteFilePromises","deleteObject","Promise","all","length","e","FAILED","name","stack","code"],"sources":["index.ts"],"sourcesContent":["import { PageImportExportTaskStatus, PbPageImportExportContext } from \"~/types\";\nimport { s3Stream } from \"../s3Stream\";\nimport { ZipOfZip } from \"../zipper\";\nimport { mockSecurity } from \"~/mockSecurity\";\nimport { SecurityIdentity } from \"@webiny/api-security/types\";\nimport { createRawEventHandler } from \"@webiny/handler-aws\";\n\nexport interface Payload {\n    taskId: string;\n    identity: SecurityIdentity;\n}\n\nexport interface Response {\n    data: string | null;\n    error: Partial<Error> | null;\n}\n\n/**\n * Handles the export pages combine workflow.\n */\nexport default () => {\n    return createRawEventHandler<Payload, PbPageImportExportContext, Response>(\n        async ({ payload, context }) => {\n            const log = console.log;\n\n            log(\"RUNNING Export Pages Combine Handler\");\n            const { pageBuilder } = context;\n            const { taskId, identity } = payload;\n\n            mockSecurity(identity, context);\n\n            try {\n                const task = await pageBuilder.pageImportExportTask.getTask(taskId);\n                if (!task) {\n                    return {\n                        data: null,\n                        error: {\n                            message: `There is no task with ID \"${taskId}\".`\n                        }\n                    };\n                }\n\n                const { exportPagesDataKey } = task.input;\n\n                // Get all files (zip) from given key\n                const listObjectResponse = await s3Stream.listObject(exportPagesDataKey);\n                if (!listObjectResponse.Contents) {\n                    return {\n                        data: null,\n                        error: {\n                            message:\n                                \"There is no Contents defined on S3 Stream while combining pages.\"\n                        }\n                    };\n                }\n\n                const zipFileKeys = listObjectResponse.Contents.filter(\n                    file => file.Key !== exportPagesDataKey\n                )\n                    .map(file => file.Key)\n                    .filter(Boolean) as string[];\n\n                // Prepare zip of all zips\n                const zipOfZip = new ZipOfZip(zipFileKeys);\n\n                // Upload\n                const pageExportUpload = await zipOfZip.process();\n                log(`Done uploading... File is located at ${pageExportUpload.Location} `);\n\n                // Update task status and save export page data key\n                await pageBuilder.pageImportExportTask.updateTask(taskId, {\n                    status: PageImportExportTaskStatus.COMPLETED,\n                    data: {\n                        message: `Finish uploading page export.`,\n                        key: pageExportUpload.Key,\n                        url: s3Stream.getPresignedUrl(pageExportUpload.Key)\n                    }\n                });\n\n                // Remove individual zip files from storage\n                const deleteFilePromises = zipFileKeys.map(key => s3Stream.deleteObject(key));\n                await Promise.all(deleteFilePromises);\n                log(`Successfully deleted ${deleteFilePromises.length} zip files.`);\n            } catch (e) {\n                log(\"[EXPORT_PAGES_COMBINE] Error => \", e);\n\n                /**\n                 * In case of error, we'll update the task status to \"failed\",\n                 * so that, client can show notify the user appropriately.\n                 */\n                await pageBuilder.pageImportExportTask.updateTask(taskId, {\n                    status: PageImportExportTaskStatus.FAILED,\n                    error: {\n                        name: e.name,\n                        message: e.message,\n                        stack: e.stack,\n                        code: \"EXPORT_FAILED\"\n                    }\n                });\n\n                return {\n                    data: null,\n                    error: {\n                        message: e.message\n                    }\n                };\n            }\n            return {\n                data: \"\",\n                error: null\n            };\n        }\n    );\n};\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AAEA;AAYA;AACA;AACA;AAFA,eAGe,MAAM;EACjB,OAAO,IAAAA,iCAAqB,EACxB,OAAO;IAAEC,OAAO;IAAEC;EAAQ,CAAC,KAAK;IAC5B,MAAMC,GAAG,GAAGC,OAAO,CAACD,GAAG;IAEvBA,GAAG,CAAC,sCAAsC,CAAC;IAC3C,MAAM;MAAEE;IAAY,CAAC,GAAGH,OAAO;IAC/B,MAAM;MAAEI,MAAM;MAAEC;IAAS,CAAC,GAAGN,OAAO;IAEpC,IAAAO,0BAAY,EAACD,QAAQ,EAAEL,OAAO,CAAC;IAE/B,IAAI;MACA,MAAMO,IAAI,GAAG,MAAMJ,WAAW,CAACK,oBAAoB,CAACC,OAAO,CAACL,MAAM,CAAC;MACnE,IAAI,CAACG,IAAI,EAAE;QACP,OAAO;UACHG,IAAI,EAAE,IAAI;UACVC,KAAK,EAAE;YACHC,OAAO,EAAG,6BAA4BR,MAAO;UACjD;QACJ,CAAC;MACL;MAEA,MAAM;QAAES;MAAmB,CAAC,GAAGN,IAAI,CAACO,KAAK;;MAEzC;MACA,MAAMC,kBAAkB,GAAG,MAAMC,kBAAQ,CAACC,UAAU,CAACJ,kBAAkB,CAAC;MACxE,IAAI,CAACE,kBAAkB,CAACG,QAAQ,EAAE;QAC9B,OAAO;UACHR,IAAI,EAAE,IAAI;UACVC,KAAK,EAAE;YACHC,OAAO,EACH;UACR;QACJ,CAAC;MACL;MAEA,MAAMO,WAAW,GAAGJ,kBAAkB,CAACG,QAAQ,CAACE,MAAM,CAClDC,IAAI,IAAIA,IAAI,CAACC,GAAG,KAAKT,kBAAkB,CAC1C,CACIU,GAAG,CAACF,IAAI,IAAIA,IAAI,CAACC,GAAG,CAAC,CACrBF,MAAM,CAACI,OAAO,CAAa;;MAEhC;MACA,MAAMC,QAAQ,GAAG,IAAIC,gBAAQ,CAACP,WAAW,CAAC;;MAE1C;MACA,MAAMQ,gBAAgB,GAAG,MAAMF,QAAQ,CAACG,OAAO,EAAE;MACjD3B,GAAG,CAAE,wCAAuC0B,gBAAgB,CAACE,QAAS,GAAE,CAAC;;MAEzE;MACA,MAAM1B,WAAW,CAACK,oBAAoB,CAACsB,UAAU,CAAC1B,MAAM,EAAE;QACtD2B,MAAM,EAAEC,iCAA0B,CAACC,SAAS;QAC5CvB,IAAI,EAAE;UACFE,OAAO,EAAG,+BAA8B;UACxCsB,GAAG,EAAEP,gBAAgB,CAACL,GAAG;UACzBa,GAAG,EAAEnB,kBAAQ,CAACoB,eAAe,CAACT,gBAAgB,CAACL,GAAG;QACtD;MACJ,CAAC,CAAC;;MAEF;MACA,MAAMe,kBAAkB,GAAGlB,WAAW,CAACI,GAAG,CAACW,GAAG,IAAIlB,kBAAQ,CAACsB,YAAY,CAACJ,GAAG,CAAC,CAAC;MAC7E,MAAMK,OAAO,CAACC,GAAG,CAACH,kBAAkB,CAAC;MACrCpC,GAAG,CAAE,wBAAuBoC,kBAAkB,CAACI,MAAO,aAAY,CAAC;IACvE,CAAC,CAAC,OAAOC,CAAC,EAAE;MACRzC,GAAG,CAAC,kCAAkC,EAAEyC,CAAC,CAAC;;MAE1C;AAChB;AACA;AACA;MACgB,MAAMvC,WAAW,CAACK,oBAAoB,CAACsB,UAAU,CAAC1B,MAAM,EAAE;QACtD2B,MAAM,EAAEC,iCAA0B,CAACW,MAAM;QACzChC,KAAK,EAAE;UACHiC,IAAI,EAAEF,CAAC,CAACE,IAAI;UACZhC,OAAO,EAAE8B,CAAC,CAAC9B,OAAO;UAClBiC,KAAK,EAAEH,CAAC,CAACG,KAAK;UACdC,IAAI,EAAE;QACV;MACJ,CAAC,CAAC;MAEF,OAAO;QACHpC,IAAI,EAAE,IAAI;QACVC,KAAK,EAAE;UACHC,OAAO,EAAE8B,CAAC,CAAC9B;QACf;MACJ,CAAC;IACL;IACA,OAAO;MACHF,IAAI,EAAE,EAAE;MACRC,KAAK,EAAE;IACX,CAAC;EACL,CAAC,CACJ;AACL,CAAC;AAAA"}

package/exportPages/process/index.js

@@ -1,192 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-var _types = require("../../types");
-var _client = require("../../client");
-var _handlerGraphql = require("@webiny/handler-graphql");
-var _utils = require("../utils");
-var _mockSecurity = require("../../mockSecurity");
-var _utils2 = require("@webiny/utils");
-var _handlerAws = require("@webiny/handler-aws");
-/**
- * Handles the export pages process workflow.
- */
-var _default = configuration => {
-  return (0, _handlerAws.createRawEventHandler)(async ({
-    payload,
-    context
-  }) => {
-    const log = console.log;
-    let subTask;
-    let noPendingTask = true;
-    let prevStatusOfSubTask = _types.PageImportExportTaskStatus.PENDING;
-    log("RUNNING Export Pages Process Handler");
-    const {
-      pageBuilder,
-      fileManager
-    } = context;
-    const {
-      taskId,
-      subTaskIndex,
-      identity
-    } = payload;
-    // Disable authorization; this is necessary because we call Page Builder CRUD methods which include authorization checks
-    // and this Lambda is invoked internally, without credentials.
-    (0, _mockSecurity.mockSecurity)(identity, context);
-    try {
-      /*
-       * Note: We're not going to DB for finding the next sub-task to process,
-       * because the data might be out of sync due to GSI eventual consistency.
-       */
-      subTask = await pageBuilder.pageImportExportTask.getSubTask(taskId, (0, _utils2.zeroPad)(subTaskIndex, 5));
-      /**
-       * Base condition!!
-       * Bail out early, if task not found or task's status is not "pending".
-       */
-      if (!subTask || subTask.status !== _types.PageImportExportTaskStatus.PENDING) {
-        noPendingTask = true;
-        return {
-          data: "",
-          error: null
-        };
-      } else {
-        noPendingTask = false;
-      }
-      log(`Fetched sub task => ${subTask.id}`);
-      const {
-        input
-      } = subTask;
-      const {
-        pageId,
-        exportPagesDataKey,
-        revisionType
-      } = input;
-
-      /**
-       * At the moment, we only export a single revision of the page.
-       * It could be "published" or "latest" depending upon user input.
-       *
-       * Note: In case of no "published" revision available, we use the latest revision.
-       */
-      let page;
-      try {
-        if (revisionType === _types.PageExportRevisionType.PUBLISHED) {
-          // Get "published" page.
-          page = await pageBuilder.getPublishedPageById({
-            id: pageId
-          });
-        } else {
-          // Get "latest" page.
-          page = await pageBuilder.getPage(pageId);
-        }
-      } catch (e) {
-        // If we're looking for "published" page and doesn't found it, get latest page.
-        if (revisionType === _types.PageExportRevisionType.PUBLISHED && e instanceof _handlerGraphql.NotFoundError) {
-          page = await pageBuilder.getPage(pageId);
-        } else {
-          throw e;
-        }
-      }
-      if (!page) {
-        log(`Unable to load page "${pageId}"`);
-        throw new _handlerGraphql.NotFoundError(`Unable to load page "${pageId}"`);
-      }
-      log(`Processing page key "${pageId}" | version ${page.version} | ${page.status}`);
-
-      // Mark task status as PROCESSING
-      subTask = await pageBuilder.pageImportExportTask.updateSubTask(taskId, subTask.id, {
-        status: _types.PageImportExportTaskStatus.PROCESSING
-      });
-      // Update stats in main task
-      await pageBuilder.pageImportExportTask.updateStats(taskId, {
-        prevStatus: prevStatusOfSubTask,
-        nextStatus: _types.PageImportExportTaskStatus.PROCESSING
-      });
-      prevStatusOfSubTask = subTask.status;
-      log(`Extracting page data and uploading to storage...`);
-      // Extract Page
-      const pageDataZip = await (0, _utils.exportPage)(page, exportPagesDataKey, fileManager);
-      log(`Finish uploading zip...`);
-      // Update task record in DB
-      subTask = await pageBuilder.pageImportExportTask.updateSubTask(taskId, subTask.id, {
-        status: _types.PageImportExportTaskStatus.COMPLETED,
-        data: {
-          message: `Finish uploading data for page "${page.id}" v${page.version} (${page.status}).`,
-          key: pageDataZip.Key
-        }
-      });
-      // Update stats in main task
-      await pageBuilder.pageImportExportTask.updateStats(taskId, {
-        prevStatus: prevStatusOfSubTask,
-        nextStatus: _types.PageImportExportTaskStatus.COMPLETED
-      });
-      prevStatusOfSubTask = subTask.status;
-    } catch (e) {
-      log("[EXPORT_PAGES_PROCESS] Error => ", e);
-      if (subTask && subTask.id) {
-        /**
-         * In case of error, we'll update the task status to "failed",
-         * so that, client can show notify the user appropriately.
-         */
-        subTask = await pageBuilder.pageImportExportTask.updateSubTask(taskId, subTask.id, {
-          status: _types.PageImportExportTaskStatus.FAILED,
-          error: {
-            name: e.name,
-            message: e.message,
-            stack: e.stack,
-            code: "EXPORT_FAILED"
-          }
-        });
-
-        // Update stats in main task
-        await pageBuilder.pageImportExportTask.updateStats(taskId, {
-          prevStatus: prevStatusOfSubTask,
-          nextStatus: _types.PageImportExportTaskStatus.FAILED
-        });
-        prevStatusOfSubTask = subTask.status;
-      }
-      return {
-        data: null,
-        error: {
-          message: e.message
-        }
-      };
-    } finally {
-      // Base condition!
-      if (noPendingTask) {
-        log(`No pending sub-task for task ${taskId}`);
-        // Combine individual page zip files.
-        await (0, _client.invokeHandlerClient)({
-          context,
-          name: configuration.handlers.combine,
-          payload: {
-            taskId,
-            identity: context.security.getIdentity()
-          },
-          description: "Export pages - combine"
-        });
-      } else {
-        console.log(`Invoking PROCESS for task "${subTaskIndex + 1}"`);
-        // We want to continue with Self invocation no matter if current page error out.
-        await (0, _client.invokeHandlerClient)({
-          context,
-          name: configuration.handlers.process,
-          payload: {
-            taskId,
-            subTaskIndex: subTaskIndex + 1,
-            identity: context.security.getIdentity()
-          },
-          description: "Export pages - process - subtask"
-        });
-      }
-    }
-    return {
-      data: "",
-      error: null
-    };
-  });
-};
-exports.default = _default;

package/exportPages/process/index.js.map

@@ -1 +0,0 @@
-{"version":3,"names":["configuration","createRawEventHandler","payload","context","log","console","subTask","noPendingTask","prevStatusOfSubTask","PageImportExportTaskStatus","PENDING","pageBuilder","fileManager","taskId","subTaskIndex","identity","mockSecurity","pageImportExportTask","getSubTask","zeroPad","status","data","error","id","input","pageId","exportPagesDataKey","revisionType","page","PageExportRevisionType","PUBLISHED","getPublishedPageById","getPage","e","NotFoundError","version","updateSubTask","PROCESSING","updateStats","prevStatus","nextStatus","pageDataZip","exportPage","COMPLETED","message","key","Key","FAILED","name","stack","code","invokeHandlerClient","handlers","combine","security","getIdentity","description","process"],"sources":["index.ts"],"sourcesContent":["import {\n    PageExportRevisionType,\n    PageImportExportTaskStatus,\n    PbPageImportExportContext\n} from \"~/types\";\nimport { invokeHandlerClient } from \"~/client\";\nimport { NotFoundError } from \"@webiny/handler-graphql\";\nimport { exportPage } from \"~/exportPages/utils\";\nimport { Payload as ExtractPayload } from \"../combine\";\nimport { mockSecurity } from \"~/mockSecurity\";\nimport { SecurityIdentity } from \"@webiny/api-security/types\";\nimport { zeroPad } from \"@webiny/utils\";\nimport { createRawEventHandler } from \"@webiny/handler-aws\";\n\ninterface Configuration {\n    handlers: {\n        process: string;\n        combine: string;\n    };\n}\n\nexport interface Payload {\n    taskId: string;\n    subTaskIndex: number;\n    identity?: SecurityIdentity;\n}\n\nexport interface Response {\n    data: string | null;\n    error: Partial<Error> | null;\n}\n\n/**\n * Handles the export pages process workflow.\n */\nexport default (configuration: Configuration) => {\n    return createRawEventHandler<Payload, PbPageImportExportContext, Response>(\n        async ({ payload, context }) => {\n            const log = console.log;\n            let subTask;\n            let noPendingTask = true;\n            let prevStatusOfSubTask = PageImportExportTaskStatus.PENDING;\n\n            log(\"RUNNING Export Pages Process Handler\");\n            const { pageBuilder, fileManager } = context;\n            const { taskId, subTaskIndex, identity } = payload;\n            // Disable authorization; this is necessary because we call Page Builder CRUD methods which include authorization checks\n            // and this Lambda is invoked internally, without credentials.\n            mockSecurity(identity as SecurityIdentity, context);\n\n            try {\n                /*\n                 * Note: We're not going to DB for finding the next sub-task to process,\n                 * because the data might be out of sync due to GSI eventual consistency.\n                 */\n                subTask = await pageBuilder.pageImportExportTask.getSubTask(\n                    taskId,\n                    zeroPad(subTaskIndex, 5)\n                );\n                /**\n                 * Base condition!!\n                 * Bail out early, if task not found or task's status is not \"pending\".\n                 */\n                if (!subTask || subTask.status !== PageImportExportTaskStatus.PENDING) {\n                    noPendingTask = true;\n                    return {\n                        data: \"\",\n                        error: null\n                    };\n                } else {\n                    noPendingTask = false;\n                }\n\n                log(`Fetched sub task => ${subTask.id}`);\n\n                const { input } = subTask;\n                const { pageId, exportPagesDataKey, revisionType } = input;\n\n                /**\n                 * At the moment, we only export a single revision of the page.\n                 * It could be \"published\" or \"latest\" depending upon user input.\n                 *\n                 * Note: In case of no \"published\" revision available, we use the latest revision.\n                 */\n                let page;\n                try {\n                    if (revisionType === PageExportRevisionType.PUBLISHED) {\n                        // Get \"published\" page.\n                        page = await pageBuilder.getPublishedPageById({ id: pageId });\n                    } else {\n                        // Get \"latest\" page.\n                        page = await pageBuilder.getPage(pageId);\n                    }\n                } catch (e) {\n                    // If we're looking for \"published\" page and doesn't found it, get latest page.\n                    if (\n                        revisionType === PageExportRevisionType.PUBLISHED &&\n                        e instanceof NotFoundError\n                    ) {\n                        page = await pageBuilder.getPage(pageId);\n                    } else {\n                        throw e;\n                    }\n                }\n\n                if (!page) {\n                    log(`Unable to load page \"${pageId}\"`);\n                    throw new NotFoundError(`Unable to load page \"${pageId}\"`);\n                }\n\n                log(`Processing page key \"${pageId}\" | version ${page.version} | ${page.status}`);\n\n                // Mark task status as PROCESSING\n                subTask = await pageBuilder.pageImportExportTask.updateSubTask(taskId, subTask.id, {\n                    status: PageImportExportTaskStatus.PROCESSING\n                });\n                // Update stats in main task\n                await pageBuilder.pageImportExportTask.updateStats(taskId, {\n                    prevStatus: prevStatusOfSubTask,\n                    nextStatus: PageImportExportTaskStatus.PROCESSING\n                });\n                prevStatusOfSubTask = subTask.status;\n\n                log(`Extracting page data and uploading to storage...`);\n                // Extract Page\n                const pageDataZip = await exportPage(page, exportPagesDataKey, fileManager);\n                log(`Finish uploading zip...`);\n                // Update task record in DB\n                subTask = await pageBuilder.pageImportExportTask.updateSubTask(taskId, subTask.id, {\n                    status: PageImportExportTaskStatus.COMPLETED,\n                    data: {\n                        message: `Finish uploading data for page \"${page.id}\" v${page.version} (${page.status}).`,\n                        key: pageDataZip.Key\n                    }\n                });\n                // Update stats in main task\n                await pageBuilder.pageImportExportTask.updateStats(taskId, {\n                    prevStatus: prevStatusOfSubTask,\n                    nextStatus: PageImportExportTaskStatus.COMPLETED\n                });\n                prevStatusOfSubTask = subTask.status;\n            } catch (e) {\n                log(\"[EXPORT_PAGES_PROCESS] Error => \", e);\n\n                if (subTask && subTask.id) {\n                    /**\n                     * In case of error, we'll update the task status to \"failed\",\n                     * so that, client can show notify the user appropriately.\n                     */\n                    subTask = await pageBuilder.pageImportExportTask.updateSubTask(\n                        taskId,\n                        subTask.id,\n                        {\n                            status: PageImportExportTaskStatus.FAILED,\n                            error: {\n                                name: e.name,\n                                message: e.message,\n                                stack: e.stack,\n                                code: \"EXPORT_FAILED\"\n                            }\n                        }\n                    );\n\n                    // Update stats in main task\n                    await pageBuilder.pageImportExportTask.updateStats(taskId, {\n                        prevStatus: prevStatusOfSubTask,\n                        nextStatus: PageImportExportTaskStatus.FAILED\n                    });\n                    prevStatusOfSubTask = subTask.status;\n                }\n\n                return {\n                    data: null,\n                    error: {\n                        message: e.message\n                    }\n                };\n            } finally {\n                // Base condition!\n                if (noPendingTask) {\n                    log(`No pending sub-task for task ${taskId}`);\n                    // Combine individual page zip files.\n                    await invokeHandlerClient<ExtractPayload>({\n                        context,\n                        name: configuration.handlers.combine,\n                        payload: {\n                            taskId,\n                            identity: context.security.getIdentity()\n                        },\n                        description: \"Export pages - combine\"\n                    });\n                } else {\n                    console.log(`Invoking PROCESS for task \"${subTaskIndex + 1}\"`);\n                    // We want to continue with Self invocation no matter if current page error out.\n                    await invokeHandlerClient<Payload>({\n                        context,\n                        name: configuration.handlers.process,\n                        payload: {\n                            taskId,\n                            subTaskIndex: subTaskIndex + 1,\n                            identity: context.security.getIdentity()\n                        },\n                        description: \"Export pages - process - subtask\"\n                    });\n                }\n            }\n            return {\n                data: \"\",\n                error: null\n            };\n        }\n    );\n};\n"],"mappings":";;;;;;AAAA;AAKA;AACA;AACA;AAEA;AAEA;AACA;AAoBA;AACA;AACA;AAFA,eAGgBA,aAA4B,IAAK;EAC7C,OAAO,IAAAC,iCAAqB,EACxB,OAAO;IAAEC,OAAO;IAAEC;EAAQ,CAAC,KAAK;IAC5B,MAAMC,GAAG,GAAGC,OAAO,CAACD,GAAG;IACvB,IAAIE,OAAO;IACX,IAAIC,aAAa,GAAG,IAAI;IACxB,IAAIC,mBAAmB,GAAGC,iCAA0B,CAACC,OAAO;IAE5DN,GAAG,CAAC,sCAAsC,CAAC;IAC3C,MAAM;MAAEO,WAAW;MAAEC;IAAY,CAAC,GAAGT,OAAO;IAC5C,MAAM;MAAEU,MAAM;MAAEC,YAAY;MAAEC;IAAS,CAAC,GAAGb,OAAO;IAClD;IACA;IACA,IAAAc,0BAAY,EAACD,QAAQ,EAAsBZ,OAAO,CAAC;IAEnD,IAAI;MACA;AAChB;AACA;AACA;MACgBG,OAAO,GAAG,MAAMK,WAAW,CAACM,oBAAoB,CAACC,UAAU,CACvDL,MAAM,EACN,IAAAM,eAAO,EAACL,YAAY,EAAE,CAAC,CAAC,CAC3B;MACD;AAChB;AACA;AACA;MACgB,IAAI,CAACR,OAAO,IAAIA,OAAO,CAACc,MAAM,KAAKX,iCAA0B,CAACC,OAAO,EAAE;QACnEH,aAAa,GAAG,IAAI;QACpB,OAAO;UACHc,IAAI,EAAE,EAAE;UACRC,KAAK,EAAE;QACX,CAAC;MACL,CAAC,MAAM;QACHf,aAAa,GAAG,KAAK;MACzB;MAEAH,GAAG,CAAE,uBAAsBE,OAAO,CAACiB,EAAG,EAAC,CAAC;MAExC,MAAM;QAAEC;MAAM,CAAC,GAAGlB,OAAO;MACzB,MAAM;QAAEmB,MAAM;QAAEC,kBAAkB;QAAEC;MAAa,CAAC,GAAGH,KAAK;;MAE1D;AAChB;AACA;AACA;AACA;AACA;MACgB,IAAII,IAAI;MACR,IAAI;QACA,IAAID,YAAY,KAAKE,6BAAsB,CAACC,SAAS,EAAE;UACnD;UACAF,IAAI,GAAG,MAAMjB,WAAW,CAACoB,oBAAoB,CAAC;YAAER,EAAE,EAAEE;UAAO,CAAC,CAAC;QACjE,CAAC,MAAM;UACH;UACAG,IAAI,GAAG,MAAMjB,WAAW,CAACqB,OAAO,CAACP,MAAM,CAAC;QAC5C;MACJ,CAAC,CAAC,OAAOQ,CAAC,EAAE;QACR;QACA,IACIN,YAAY,KAAKE,6BAAsB,CAACC,SAAS,IACjDG,CAAC,YAAYC,6BAAa,EAC5B;UACEN,IAAI,GAAG,MAAMjB,WAAW,CAACqB,OAAO,CAACP,MAAM,CAAC;QAC5C,CAAC,MAAM;UACH,MAAMQ,CAAC;QACX;MACJ;MAEA,IAAI,CAACL,IAAI,EAAE;QACPxB,GAAG,CAAE,wBAAuBqB,MAAO,GAAE,CAAC;QACtC,MAAM,IAAIS,6BAAa,CAAE,wBAAuBT,MAAO,GAAE,CAAC;MAC9D;MAEArB,GAAG,CAAE,wBAAuBqB,MAAO,eAAcG,IAAI,CAACO,OAAQ,MAAKP,IAAI,CAACR,MAAO,EAAC,CAAC;;MAEjF;MACAd,OAAO,GAAG,MAAMK,WAAW,CAACM,oBAAoB,CAACmB,aAAa,CAACvB,MAAM,EAAEP,OAAO,CAACiB,EAAE,EAAE;QAC/EH,MAAM,EAAEX,iCAA0B,CAAC4B;MACvC,CAAC,CAAC;MACF;MACA,MAAM1B,WAAW,CAACM,oBAAoB,CAACqB,WAAW,CAACzB,MAAM,EAAE;QACvD0B,UAAU,EAAE/B,mBAAmB;QAC/BgC,UAAU,EAAE/B,iCAA0B,CAAC4B;MAC3C,CAAC,CAAC;MACF7B,mBAAmB,GAAGF,OAAO,CAACc,MAAM;MAEpChB,GAAG,CAAE,kDAAiD,CAAC;MACvD;MACA,MAAMqC,WAAW,GAAG,MAAM,IAAAC,iBAAU,EAACd,IAAI,EAAEF,kBAAkB,EAAEd,WAAW,CAAC;MAC3ER,GAAG,CAAE,yBAAwB,CAAC;MAC9B;MACAE,OAAO,GAAG,MAAMK,WAAW,CAACM,oBAAoB,CAACmB,aAAa,CAACvB,MAAM,EAAEP,OAAO,CAACiB,EAAE,EAAE;QAC/EH,MAAM,EAAEX,iCAA0B,CAACkC,SAAS;QAC5CtB,IAAI,EAAE;UACFuB,OAAO,EAAG,mCAAkChB,IAAI,CAACL,EAAG,MAAKK,IAAI,CAACO,OAAQ,KAAIP,IAAI,CAACR,MAAO,IAAG;UACzFyB,GAAG,EAAEJ,WAAW,CAACK;QACrB;MACJ,CAAC,CAAC;MACF;MACA,MAAMnC,WAAW,CAACM,oBAAoB,CAACqB,WAAW,CAACzB,MAAM,EAAE;QACvD0B,UAAU,EAAE/B,mBAAmB;QAC/BgC,UAAU,EAAE/B,iCAA0B,CAACkC;MAC3C,CAAC,CAAC;MACFnC,mBAAmB,GAAGF,OAAO,CAACc,MAAM;IACxC,CAAC,CAAC,OAAOa,CAAC,EAAE;MACR7B,GAAG,CAAC,kCAAkC,EAAE6B,CAAC,CAAC;MAE1C,IAAI3B,OAAO,IAAIA,OAAO,CAACiB,EAAE,EAAE;QACvB;AACpB;AACA;AACA;QACoBjB,OAAO,GAAG,MAAMK,WAAW,CAACM,oBAAoB,CAACmB,aAAa,CAC1DvB,MAAM,EACNP,OAAO,CAACiB,EAAE,EACV;UACIH,MAAM,EAAEX,iCAA0B,CAACsC,MAAM;UACzCzB,KAAK,EAAE;YACH0B,IAAI,EAAEf,CAAC,CAACe,IAAI;YACZJ,OAAO,EAAEX,CAAC,CAACW,OAAO;YAClBK,KAAK,EAAEhB,CAAC,CAACgB,KAAK;YACdC,IAAI,EAAE;UACV;QACJ,CAAC,CACJ;;QAED;QACA,MAAMvC,WAAW,CAACM,oBAAoB,CAACqB,WAAW,CAACzB,MAAM,EAAE;UACvD0B,UAAU,EAAE/B,mBAAmB;UAC/BgC,UAAU,EAAE/B,iCAA0B,CAACsC;QAC3C,CAAC,CAAC;QACFvC,mBAAmB,GAAGF,OAAO,CAACc,MAAM;MACxC;MAEA,OAAO;QACHC,IAAI,EAAE,IAAI;QACVC,KAAK,EAAE;UACHsB,OAAO,EAAEX,CAAC,CAACW;QACf;MACJ,CAAC;IACL,CAAC,SAAS;MACN;MACA,IAAIrC,aAAa,EAAE;QACfH,GAAG,CAAE,gCAA+BS,MAAO,EAAC,CAAC;QAC7C;QACA,MAAM,IAAAsC,2BAAmB,EAAiB;UACtChD,OAAO;UACP6C,IAAI,EAAEhD,aAAa,CAACoD,QAAQ,CAACC,OAAO;UACpCnD,OAAO,EAAE;YACLW,MAAM;YACNE,QAAQ,EAAEZ,OAAO,CAACmD,QAAQ,CAACC,WAAW;UAC1C,CAAC;UACDC,WAAW,EAAE;QACjB,CAAC,CAAC;MACN,CAAC,MAAM;QACHnD,OAAO,CAACD,GAAG,CAAE,8BAA6BU,YAAY,GAAG,CAAE,GAAE,CAAC;QAC9D;QACA,MAAM,IAAAqC,2BAAmB,EAAU;UAC/BhD,OAAO;UACP6C,IAAI,EAAEhD,aAAa,CAACoD,QAAQ,CAACK,OAAO;UACpCvD,OAAO,EAAE;YACLW,MAAM;YACNC,YAAY,EAAEA,YAAY,GAAG,CAAC;YAC9BC,QAAQ,EAAEZ,OAAO,CAACmD,QAAQ,CAACC,WAAW;UAC1C,CAAC;UACDC,WAAW,EAAE;QACjB,CAAC,CAAC;MACN;IACJ;IACA,OAAO;MACHnC,IAAI,EAAE,EAAE;MACRC,KAAK,EAAE;IACX,CAAC;EACL,CAAC,CACJ;AACL,CAAC;AAAA"}

package/exportPages/s3Stream.d.ts

@@ -1,31 +0,0 @@
-/// <reference types="node" />
-/// <reference types="node" />
-import { PassThrough, Readable } from "stream";
-import S3 from "aws-sdk/clients/s3";
-declare class S3Stream {
-    s3: S3;
-    bucket: string;
-    constructor();
-    getPresignedUrl(key: string): string;
-    /**
-     * We're checking if the file is accessible on S3 by getting object meta data.
-     * It help us to filter files that we need to download as part of export data.
-     * @param Key {string}
-     */
-    isFileAccessible(Key: string): Promise<boolean>;
-    getObjectHead(Key: string): Promise<S3.HeadObjectOutput>;
-    readStream(Key: string): Readable;
-    writeStream(Key: string, contentType?: string): {
-        streamPassThrough: PassThrough;
-        streamPassThroughUploadPromise: Promise<S3.ManagedUpload.SendData>;
-    };
-    upload(params: {
-        Key: string;
-        ContentType: string;
-        Body: Buffer;
-    }): Promise<S3.ManagedUpload.SendData>;
-    listObject(prefix: string): Promise<S3.ListObjectsOutput>;
-    deleteObject(key: string): Promise<S3.DeleteObjectOutput>;
-}
-export declare const s3Stream: S3Stream;
-export {};

package/exportPages/s3Stream.js.map

@@ -1 +0,0 @@
-{"version":3,"names":["ARCHIVE_CONTENT_TYPE","S3Stream","constructor","s3","S3","region","process","env","AWS_REGION","bucket","S3_BUCKET","getPresignedUrl","key","getSignedUrl","Bucket","Key","Expires","isFileAccessible","getObjectHead","error","console","warn","log","headObject","promise","readStream","getObject","createReadStream","writeStream","contentType","streamPassThrough","Stream","PassThrough","params","ACL","Body","ContentType","streamPassThroughUploadPromise","upload","listObject","prefix","listObjects","Prefix","deleteObject","s3Stream"],"sources":["s3Stream.ts"],"sourcesContent":["import { Stream, PassThrough, Readable } from \"stream\";\nimport S3 from \"aws-sdk/clients/s3\";\n\nconst ARCHIVE_CONTENT_TYPE = \"application/zip\";\n\nclass S3Stream {\n    s3: S3;\n    bucket: string;\n\n    constructor() {\n        this.s3 = new S3({\n            region: process.env.AWS_REGION as string\n        });\n        this.bucket = process.env.S3_BUCKET as string;\n    }\n\n    getPresignedUrl(key: string) {\n        return this.s3.getSignedUrl(\"getObject\", {\n            Bucket: this.bucket,\n            Key: key,\n            Expires: 604800 // 1 week\n        });\n    }\n\n    /**\n     * We're checking if the file is accessible on S3 by getting object meta data.\n     * It help us to filter files that we need to download as part of export data.\n     * @param Key {string}\n     */\n    async isFileAccessible(Key: string): Promise<boolean> {\n        try {\n            await this.getObjectHead(Key);\n            return true;\n        } catch (error) {\n            console.warn(`Error while fetching meta data for file \"${Key}\"`);\n            console.log(error);\n            return false;\n        }\n    }\n\n    getObjectHead(Key: string): Promise<S3.HeadObjectOutput> {\n        return this.s3.headObject({ Bucket: this.bucket, Key }).promise();\n    }\n\n    readStream(Key: string): Readable {\n        return this.s3.getObject({ Bucket: this.bucket, Key }).createReadStream();\n    }\n\n    writeStream(\n        Key: string,\n        contentType: string = ARCHIVE_CONTENT_TYPE\n    ): {\n        streamPassThrough: PassThrough;\n        streamPassThroughUploadPromise: Promise<S3.ManagedUpload.SendData>;\n    } {\n        const streamPassThrough = new Stream.PassThrough();\n\n        const params: S3.PutObjectRequest = {\n            ACL: \"private\",\n            Body: streamPassThrough,\n            Bucket: this.bucket,\n            ContentType: contentType,\n            Key\n        };\n\n        return {\n            streamPassThrough: streamPassThrough,\n            /**\n             * We're not using the `FileManager` storage plugin here because it currently doesn't support streams.\n             */\n            streamPassThroughUploadPromise: this.s3.upload(params).promise()\n        };\n    }\n\n    upload(params: {\n        Key: string;\n        ContentType: string;\n        Body: Buffer;\n    }): Promise<S3.ManagedUpload.SendData> {\n        return this.s3\n            .upload({\n                ACL: \"private\",\n                Bucket: this.bucket,\n                ...params\n            })\n            .promise();\n    }\n\n    listObject(prefix: string): Promise<S3.ListObjectsOutput> {\n        return this.s3\n            .listObjects({\n                Bucket: this.bucket,\n                Prefix: prefix\n            })\n            .promise();\n    }\n\n    deleteObject(key: string): Promise<S3.DeleteObjectOutput> {\n        return this.s3.deleteObject({ Key: key, Bucket: this.bucket }).promise();\n    }\n}\n\nexport const s3Stream = new S3Stream();\n"],"mappings":";;;;;;;;;AAAA;AACA;AAEA,MAAMA,oBAAoB,GAAG,iBAAiB;AAE9C,MAAMC,QAAQ,CAAC;EAIXC,WAAW,GAAG;IAAA;IAAA;IACV,IAAI,CAACC,EAAE,GAAG,IAAIC,UAAE,CAAC;MACbC,MAAM,EAAEC,OAAO,CAACC,GAAG,CAACC;IACxB,CAAC,CAAC;IACF,IAAI,CAACC,MAAM,GAAGH,OAAO,CAACC,GAAG,CAACG,SAAmB;EACjD;EAEAC,eAAe,CAACC,GAAW,EAAE;IACzB,OAAO,IAAI,CAACT,EAAE,CAACU,YAAY,CAAC,WAAW,EAAE;MACrCC,MAAM,EAAE,IAAI,CAACL,MAAM;MACnBM,GAAG,EAAEH,GAAG;MACRI,OAAO,EAAE,MAAM,CAAC;IACpB,CAAC,CAAC;EACN;;EAEA;AACJ;AACA;AACA;AACA;EACI,MAAMC,gBAAgB,CAACF,GAAW,EAAoB;IAClD,IAAI;MACA,MAAM,IAAI,CAACG,aAAa,CAACH,GAAG,CAAC;MAC7B,OAAO,IAAI;IACf,CAAC,CAAC,OAAOI,KAAK,EAAE;MACZC,OAAO,CAACC,IAAI,CAAE,4CAA2CN,GAAI,GAAE,CAAC;MAChEK,OAAO,CAACE,GAAG,CAACH,KAAK,CAAC;MAClB,OAAO,KAAK;IAChB;EACJ;EAEAD,aAAa,CAACH,GAAW,EAAgC;IACrD,OAAO,IAAI,CAACZ,EAAE,CAACoB,UAAU,CAAC;MAAET,MAAM,EAAE,IAAI,CAACL,MAAM;MAAEM;IAAI,CAAC,CAAC,CAACS,OAAO,EAAE;EACrE;EAEAC,UAAU,CAACV,GAAW,EAAY;IAC9B,OAAO,IAAI,CAACZ,EAAE,CAACuB,SAAS,CAAC;MAAEZ,MAAM,EAAE,IAAI,CAACL,MAAM;MAAEM;IAAI,CAAC,CAAC,CAACY,gBAAgB,EAAE;EAC7E;EAEAC,WAAW,CACPb,GAAW,EACXc,WAAmB,GAAG7B,oBAAoB,EAI5C;IACE,MAAM8B,iBAAiB,GAAG,IAAIC,cAAM,CAACC,WAAW,EAAE;IAElD,MAAMC,MAA2B,GAAG;MAChCC,GAAG,EAAE,SAAS;MACdC,IAAI,EAAEL,iBAAiB;MACvBhB,MAAM,EAAE,IAAI,CAACL,MAAM;MACnB2B,WAAW,EAAEP,WAAW;MACxBd;IACJ,CAAC;IAED,OAAO;MACHe,iBAAiB,EAAEA,iBAAiB;MACpC;AACZ;AACA;MACYO,8BAA8B,EAAE,IAAI,CAAClC,EAAE,CAACmC,MAAM,CAACL,MAAM,CAAC,CAACT,OAAO;IAClE,CAAC;EACL;EAEAc,MAAM,CAACL,MAIN,EAAsC;IACnC,OAAO,IAAI,CAAC9B,EAAE,CACTmC,MAAM;MACHJ,GAAG,EAAE,SAAS;MACdpB,MAAM,EAAE,IAAI,CAACL;IAAM,GAChBwB,MAAM,EACX,CACDT,OAAO,EAAE;EAClB;EAEAe,UAAU,CAACC,MAAc,EAAiC;IACtD,OAAO,IAAI,CAACrC,EAAE,CACTsC,WAAW,CAAC;MACT3B,MAAM,EAAE,IAAI,CAACL,MAAM;MACnBiC,MAAM,EAAEF;IACZ,CAAC,CAAC,CACDhB,OAAO,EAAE;EAClB;EAEAmB,YAAY,CAAC/B,GAAW,EAAkC;IACtD,OAAO,IAAI,CAACT,EAAE,CAACwC,YAAY,CAAC;MAAE5B,GAAG,EAAEH,GAAG;MAAEE,MAAM,EAAE,IAAI,CAACL;IAAO,CAAC,CAAC,CAACe,OAAO,EAAE;EAC5E;AACJ;AAEO,MAAMoB,QAAQ,GAAG,IAAI3C,QAAQ,EAAE;AAAC"}

package/exportPages/utils.d.ts

@@ -1,13 +0,0 @@
-import S3 from "aws-sdk/clients/s3";
-import { Page, File } from "@webiny/api-page-builder/types";
-import { FileManagerContext } from "@webiny/api-file-manager/types";
-export declare const EXPORT_PAGES_FOLDER_KEY = "WEBINY_PB_EXPORT_PAGES";
-export interface ExportedPageData {
-    page: Pick<Page, "content" | "title" | "version" | "status" | "settings" | "path">;
-    files: ImageFile[];
-}
-export declare function exportPage(page: Page, exportPagesDataKey: string, fileManager: FileManagerContext["fileManager"]): Promise<S3.ManagedUpload.SendData>;
-export interface ImageFile extends Omit<File, "src"> {
-    key: string;
-}
-export declare function extractFilesFromPageData(data: Record<string, any>, files?: any[]): ImageFile[];

package/exportPages/utils.js

@@ -1,100 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.EXPORT_PAGES_FOLDER_KEY = void 0;
-exports.exportPage = exportPage;
-exports.extractFilesFromPageData = extractFilesFromPageData;
-var _get = _interopRequireDefault(require("lodash/get"));
-var _s3Stream = require("./s3Stream");
-var _zipper = _interopRequireDefault(require("./zipper"));
-const EXPORT_PAGES_FOLDER_KEY = "WEBINY_PB_EXPORT_PAGES";
-exports.EXPORT_PAGES_FOLDER_KEY = EXPORT_PAGES_FOLDER_KEY;
-async function getFilteredFiles(files) {
-  const uniqueFileKeys = new Map();
-  const promises = files.map(file => _s3Stream.s3Stream.isFileAccessible(file.key));
-  const isFileAvailableResults = await Promise.all(promises);
-  const filesAvailableForDownload = [];
-  // Filter files
-  for (let i = 0; i < files.length; i++) {
-    const file = files[i];
-    // Check file accessibility
-    if (isFileAvailableResults[i] && !uniqueFileKeys.has(file.key)) {
-      filesAvailableForDownload.push(file);
-      uniqueFileKeys.set(file.key, true);
-    }
-  }
-  return filesAvailableForDownload;
-}
-async function exportPage(page, exportPagesDataKey, fileManager) {
-  // Extract all files
-  const files = extractFilesFromPageData(page.content || {});
-  // Filter files
-  const filesAvailableForDownload = await getFilteredFiles(files);
-  // Extract images from page settings
-  const pageSettingsImages = [(0, _get.default)(page, "settings.general.image"), (0, _get.default)(page, "settings.social.image")].filter(image => image && image.src);
-  const pageSettingsImagesData = [];
-  // Get file data for all images inside "page.settings"
-  for (let i = 0; i < pageSettingsImages.length; i++) {
-    const {
-      id
-    } = pageSettingsImages[i];
-    const file = await fileManager.files.getFile(id);
-    pageSettingsImagesData.push(file);
-  }
-
-  // Extract the page data in a json file and upload it to S3
-  const pageData = {
-    page: {
-      content: page.content,
-      title: page.title,
-      path: page.path,
-      version: page.version,
-      status: page.status,
-      settings: page.settings
-    },
-    files: [...filesAvailableForDownload, ...pageSettingsImagesData]
-  };
-  const pageDataBuffer = Buffer.from(JSON.stringify(pageData));
-  const zipper = new _zipper.default({
-    exportInfo: {
-      files: [...filesAvailableForDownload, ...pageSettingsImagesData],
-      pageTitle: page.title,
-      pageDataBuffer
-    },
-    archiveFileKey: exportPagesDataKey
-  });
-  return zipper.process();
-}
-function extractFilesFromPageData(data, files = []) {
-  // Base case: termination
-  if (!data || typeof data !== "object") {
-    return files;
-  }
-  // Recursively call function for each element
-  if (Array.isArray(data)) {
-    for (let i = 0; i < data.length; i++) {
-      const element = data[i];
-      extractFilesFromPageData(element, files);
-    }
-    return files;
-  }
-
-  // Main
-  const tuple = Object.entries(data);
-  for (let i = 0; i < tuple.length; i++) {
-    const [key, value] = tuple[i];
-    // TODO: @ashutosh extract it to plugins, so that, we can handle cases for other components too.
-    if (key === "file" && value) {
-      files.push(value);
-    } else if (key === "images" && Array.isArray(value)) {
-      // Handle case for "images-list" component
-      files.push(...value);
-    } else {
-      extractFilesFromPageData(value, files);
-    }
-  }
-  return files;
-}