@webiny/pulumi-aws 0.0.0-ee-vpcs.549378cf03

package/utils/uploadFolderToS3.js

@@ -0,0 +1,188 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.uploadFolderToS3 = void 0;
+
+var _objectSpread2 = _interopRequireDefault(require("@babel/runtime/helpers/objectSpread2"));
+
+var _fs = _interopRequireDefault(require("fs"));
+
+var _nodeFetch = _interopRequireDefault(require("node-fetch"));
+
+var _formData = _interopRequireDefault(require("form-data"));
+
+var _s = _interopRequireDefault(require("aws-sdk/clients/s3"));
+
+var _mime = _interopRequireDefault(require("mime"));
+
+var _chunk = _interopRequireDefault(require("lodash/chunk"));
+
+var _path = require("path");
+
+var _crawlDirectory = require("./crawlDirectory");
+
+var _getPresignedPost = require("./getPresignedPost");
+
+function getFileChecksum(file) {
+  const crypto = require("crypto");
+
+  const hash = crypto.createHash("md5");
+  return new Promise(resolve => {
+    const stream = _fs.default.createReadStream(file);
+
+    stream.on("data", function (data) {
+      hash.update(data, "utf8");
+    });
+    stream.on("end", function () {
+      resolve(hash.digest("hex"));
+    });
+  });
+}
+
+const uploadFolderToS3 = async ({
+  path: root,
+  bucket,
+  onFileUploadSuccess,
+  onFileUploadError,
+  onFileUploadSkip,
+  // For backwards compatibility, we need to leave ACL on "public-read".
+  acl = "public-read",
+  cacheControl = "max-age=31536000"
+}) => {
+  const s3 = new _s.default({
+    region: process.env.AWS_REGION
+  });
+
+  if (!_fs.default.existsSync(root)) {
+    throw new Error("Cannot continue, folder does not exist.");
+  }
+
+  const paths = [];
+  await (0, _crawlDirectory.crawlDirectory)(root, async path => {
+    paths.push(path);
+  });
+  const pathsChunks = (0, _chunk.default)(paths, 20);
+  const cacheControls = [];
+
+  if (typeof cacheControl === "string") {
+    cacheControls.push({
+      pattern: /.*/,
+      value: cacheControl
+    });
+  } else if (Array.isArray(cacheControls)) {
+    cacheControls.push(...cacheControl);
+  }
+
+  for (let i = 0; i < pathsChunks.length; i++) {
+    const chunk = pathsChunks[i];
+    const promises = [];
+
+    for (let j = 0; j < chunk.length; j++) {
+      const path = chunk[j];
+      promises.push(new Promise(async resolve => {
+        // We also replace "\" with "/", which is a path separator on Windows' CMD or Powershell.
+        const key = (0, _path.relative)(root, path).replace(/\\/g, "/");
+
+        try {
+          // Get file checksum so that we can check if a file needs to be uploaded or not.
+          const checksum = await getFileChecksum(path);
+          let skipUpload = false;
+
+          try {
+            var _existingObject$Metad;
+
+            const existingObject = await s3.headObject({
+              Bucket: bucket,
+              Key: key
+            }).promise();
+
+            if (((_existingObject$Metad = existingObject.Metadata) === null || _existingObject$Metad === void 0 ? void 0 : _existingObject$Metad.checksum) === checksum) {
+              skipUpload = true;
+            }
+          } catch {// Do nothing.
+          }
+
+          if (skipUpload) {
+            if (typeof onFileUploadSkip === "function") {
+              await onFileUploadSkip({
+                paths: {
+                  full: path,
+                  relative: key
+                }
+              });
+            }
+          } else {
+            const cacheControl = cacheControls.find(item => item.pattern.test(key));
+
+            const contentType = _mime.default.getType(path);
+
+            const {
+              url,
+              fields
+            } = await (0, _getPresignedPost.getPresignedPost)({
+              bucket,
+              key,
+              acl,
+              checksum,
+              contentType,
+              cacheControl: cacheControl ? cacheControl.value : undefined
+            });
+            const data = (0, _objectSpread2.default)((0, _objectSpread2.default)({}, fields), {}, {
+              "Content-Type": contentType || "",
+              "X-Amz-Meta-Checksum": checksum,
+              file: _fs.default.readFileSync(path)
+            });
+
+            if (cacheControl) {
+              data["Cache-Control"] = cacheControl.value;
+            }
+
+            const formData = new _formData.default();
+            Object.keys(data).forEach(key => {
+              formData.append(key, data[key]);
+            });
+            const res = await (0, _nodeFetch.default)(url, {
+              method: "POST",
+              body: formData
+            });
+
+            if (res.status > 299) {
+              throw new Error(`${res.statusText}\n${await res.text()}`);
+            }
+
+            if (typeof onFileUploadSuccess === "function") {
+              await onFileUploadSuccess({
+                paths: {
+                  full: path,
+                  relative: key
+                }
+              });
+            }
+          }
+
+          resolve();
+        } catch (e) {
+          if (typeof onFileUploadError === "function") {
+            await onFileUploadError({
+              paths: {
+                full: path,
+                relative: key
+              },
+              error: e
+            });
+          }
+
+          resolve();
+        }
+      }));
+    }
+
+    await Promise.all(promises);
+  }
+};
+
+exports.uploadFolderToS3 = uploadFolderToS3;

package/utils/uploadFolderToS3.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["getFileChecksum","file","crypto","require","hash","createHash","Promise","resolve","stream","fs","createReadStream","on","data","update","digest","uploadFolderToS3","path","root","bucket","onFileUploadSuccess","onFileUploadError","onFileUploadSkip","acl","cacheControl","s3","S3Client","region","process","env","AWS_REGION","existsSync","Error","paths","crawlDirectory","push","pathsChunks","chunk","cacheControls","pattern","value","Array","isArray","i","length","promises","j","key","relative","replace","checksum","skipUpload","existingObject","headObject","Bucket","Key","promise","Metadata","full","find","item","test","contentType","mime","getType","url","fields","getPresignedPost","undefined","readFileSync","formData","FormData","Object","keys","forEach","append","res","fetch","method","body","status","statusText","text","e","error","all"],"sources":["uploadFolderToS3.ts"],"sourcesContent":["import fs from \"fs\";\nimport fetch from \"node-fetch\";\nimport FormData from \"form-data\";\nimport S3Client from \"aws-sdk/clients/s3\";\nimport mime from \"mime\";\nimport chunk from \"lodash/chunk\";\nimport { relative } from \"path\";\nimport { crawlDirectory } from \"./crawlDirectory\";\nimport { getPresignedPost } from \"./getPresignedPost\";\n\nfunction getFileChecksum(file: string): Promise<string> {\n    const crypto = require(\"crypto\");\n    const hash = crypto.createHash(\"md5\");\n\n    return new Promise(resolve => {\n        const stream = fs.createReadStream(file);\n        stream.on(\"data\", function (data: any) {\n            hash.update(data, \"utf8\");\n        });\n\n        stream.on(\"end\", function () {\n            resolve(hash.digest(\"hex\"));\n        });\n    });\n}\n\nimport { BucketName, CacheControl } from \"aws-sdk/clients/s3\";\n\nexport interface Paths {\n    full: string;\n    relative: string;\n}\n\nexport type CacheControls = Array<{ pattern: RegExp; value: CacheControl }>;\n\nexport interface UploadFolderToS3Params {\n    // Path to the folder that needs to be uploaded.\n    path: string;\n\n    // Object ACL.\n    acl: string;\n\n    // A callback that gets called every time a file has been uploaded successfully.\n    onFileUploadSuccess: (params: { paths: Paths }) => void;\n\n    // A callback that gets called every time a file has not been uploaded successfully.\n    onFileUploadError: (params: { paths: Paths; error: Error }) => void;\n\n    // A callback that gets called every time a file upload has been skipped.\n    onFileUploadSkip: (params: { paths: Paths }) => void;\n\n    // Target bucket\n    bucket: BucketName;\n\n    // Cache control to apply to each uploaded file\n    cacheControl?: CacheControl | CacheControls;\n}\n\nexport const uploadFolderToS3 = async ({\n    path: root,\n    bucket,\n    onFileUploadSuccess,\n    onFileUploadError,\n    onFileUploadSkip,\n    // For backwards compatibility, we need to leave ACL on \"public-read\".\n    acl = \"public-read\",\n    cacheControl = \"max-age=31536000\"\n}: UploadFolderToS3Params) => {\n    const s3 = new S3Client({ region: process.env.AWS_REGION });\n\n    if (!fs.existsSync(root)) {\n        throw new Error(\"Cannot continue, folder does not exist.\");\n    }\n\n    const paths: string[] = [];\n\n    await crawlDirectory(root, async (path: string) => {\n        paths.push(path);\n    });\n\n    const pathsChunks = chunk(paths, 20);\n\n    const cacheControls: CacheControls = [];\n    if (typeof cacheControl === \"string\") {\n        cacheControls.push({ pattern: /.*/, value: cacheControl });\n    } else if (Array.isArray(cacheControls)) {\n        cacheControls.push(...cacheControl);\n    }\n\n    for (let i = 0; i < pathsChunks.length; i++) {\n        const chunk = pathsChunks[i];\n\n        const promises = [];\n        for (let j = 0; j < chunk.length; j++) {\n            const path = chunk[j];\n\n            promises.push(\n                new Promise<void>(async resolve => {\n                    // We also replace \"\\\" with \"/\", which is a path separator on Windows' CMD or Powershell.\n                    const key = relative(root, path).replace(/\\\\/g, \"/\");\n                    try {\n                        // Get file checksum so that we can check if a file needs to be uploaded or not.\n                        const checksum = await getFileChecksum(path);\n\n                        let skipUpload = false;\n                        try {\n                            const existingObject = await s3\n                                .headObject({\n                                    Bucket: bucket,\n                                    Key: key\n                                })\n                                .promise();\n\n                            if (existingObject.Metadata?.checksum === checksum) {\n                                skipUpload = true;\n                            }\n                        } catch {\n                            // Do nothing.\n                        }\n\n                        if (skipUpload) {\n                            if (typeof onFileUploadSkip === \"function\") {\n                                await onFileUploadSkip({ paths: { full: path, relative: key } });\n                            }\n                        } else {\n                            const cacheControl = cacheControls.find(item => item.pattern.test(key));\n                            const contentType = mime.getType(path);\n\n                            const { url, fields } = await getPresignedPost({\n                                bucket,\n                                key,\n                                acl,\n                                checksum,\n                                contentType,\n                                cacheControl: cacheControl ? cacheControl.value : undefined\n                            });\n\n                            const data: Record<string, string | Buffer> = {\n                                ...fields,\n                                \"Content-Type\": contentType || \"\",\n                                \"X-Amz-Meta-Checksum\": checksum,\n                                file: fs.readFileSync(path)\n                            };\n\n                            if (cacheControl) {\n                                data[\"Cache-Control\"] = cacheControl.value;\n                            }\n\n                            const formData = new FormData();\n                            Object.keys(data).forEach(key => {\n                                formData.append(key, data[key]);\n                            });\n\n                            const res = await fetch(url, {\n                                method: \"POST\",\n                                body: formData\n                            });\n\n                            if (res.status > 299) {\n                                throw new Error(`${res.statusText}\\n${await res.text()}`);\n                            }\n\n                            if (typeof onFileUploadSuccess === \"function\") {\n                                await onFileUploadSuccess({ paths: { full: path, relative: key } });\n                            }\n                        }\n                        resolve();\n                    } catch (e) {\n                        if (typeof onFileUploadError === \"function\") {\n                            await onFileUploadError({\n                                paths: { full: path, relative: key },\n                                error: e\n                            });\n                        }\n                        resolve();\n                    }\n                })\n            );\n        }\n\n        await Promise.all(promises);\n    }\n};\n"],"mappings":";;;;;;;;;;;AAAA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AAEA,SAASA,eAAT,CAAyBC,IAAzB,EAAwD;EACpD,MAAMC,MAAM,GAAGC,OAAO,CAAC,QAAD,CAAtB;;EACA,MAAMC,IAAI,GAAGF,MAAM,CAACG,UAAP,CAAkB,KAAlB,CAAb;EAEA,OAAO,IAAIC,OAAJ,CAAYC,OAAO,IAAI;IAC1B,MAAMC,MAAM,GAAGC,WAAA,CAAGC,gBAAH,CAAoBT,IAApB,CAAf;;IACAO,MAAM,CAACG,EAAP,CAAU,MAAV,EAAkB,UAAUC,IAAV,EAAqB;MACnCR,IAAI,CAACS,MAAL,CAAYD,IAAZ,EAAkB,MAAlB;IACH,CAFD;IAIAJ,MAAM,CAACG,EAAP,CAAU,KAAV,EAAiB,YAAY;MACzBJ,OAAO,CAACH,IAAI,CAACU,MAAL,CAAY,KAAZ,CAAD,CAAP;IACH,CAFD;EAGH,CATM,CAAP;AAUH;;AAkCM,MAAMC,gBAAgB,GAAG,OAAO;EACnCC,IAAI,EAAEC,IAD6B;EAEnCC,MAFmC;EAGnCC,mBAHmC;EAInCC,iBAJmC;EAKnCC,gBALmC;EAMnC;EACAC,GAAG,GAAG,aAP6B;EAQnCC,YAAY,GAAG;AARoB,CAAP,KASF;EAC1B,MAAMC,EAAE,GAAG,IAAIC,UAAJ,CAAa;IAAEC,MAAM,EAAEC,OAAO,CAACC,GAAR,CAAYC;EAAtB,CAAb,CAAX;;EAEA,IAAI,CAACpB,WAAA,CAAGqB,UAAH,CAAcb,IAAd,CAAL,EAA0B;IACtB,MAAM,IAAIc,KAAJ,CAAU,yCAAV,CAAN;EACH;;EAED,MAAMC,KAAe,GAAG,EAAxB;EAEA,MAAM,IAAAC,8BAAA,EAAehB,IAAf,EAAqB,MAAOD,IAAP,IAAwB;IAC/CgB,KAAK,CAACE,IAAN,CAAWlB,IAAX;EACH,CAFK,CAAN;EAIA,MAAMmB,WAAW,GAAG,IAAAC,cAAA,EAAMJ,KAAN,EAAa,EAAb,CAApB;EAEA,MAAMK,aAA4B,GAAG,EAArC;;EACA,IAAI,OAAOd,YAAP,KAAwB,QAA5B,EAAsC;IAClCc,aAAa,CAACH,IAAd,CAAmB;MAAEI,OAAO,EAAE,IAAX;MAAiBC,KAAK,EAAEhB;IAAxB,CAAnB;EACH,CAFD,MAEO,IAAIiB,KAAK,CAACC,OAAN,CAAcJ,aAAd,CAAJ,EAAkC;IACrCA,aAAa,CAACH,IAAd,CAAmB,GAAGX,YAAtB;EACH;;EAED,KAAK,IAAImB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGP,WAAW,CAACQ,MAAhC,EAAwCD,CAAC,EAAzC,EAA6C;IACzC,MAAMN,KAAK,GAAGD,WAAW,CAACO,CAAD,CAAzB;IAEA,MAAME,QAAQ,GAAG,EAAjB;;IACA,KAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGT,KAAK,CAACO,MAA1B,EAAkCE,CAAC,EAAnC,EAAuC;MACnC,MAAM7B,IAAI,GAAGoB,KAAK,CAACS,CAAD,CAAlB;MAEAD,QAAQ,CAACV,IAAT,CACI,IAAI5B,OAAJ,CAAkB,MAAMC,OAAN,IAAiB;QAC/B;QACA,MAAMuC,GAAG,GAAG,IAAAC,cAAA,EAAS9B,IAAT,EAAeD,IAAf,EAAqBgC,OAArB,CAA6B,KAA7B,EAAoC,GAApC,CAAZ;;QACA,IAAI;UACA;UACA,MAAMC,QAAQ,GAAG,MAAMjD,eAAe,CAACgB,IAAD,CAAtC;UAEA,IAAIkC,UAAU,GAAG,KAAjB;;UACA,IAAI;YAAA;;YACA,MAAMC,cAAc,GAAG,MAAM3B,EAAE,CAC1B4B,UADwB,CACb;cACRC,MAAM,EAAEnC,MADA;cAERoC,GAAG,EAAER;YAFG,CADa,EAKxBS,OALwB,EAA7B;;YAOA,IAAI,0BAAAJ,cAAc,CAACK,QAAf,gFAAyBP,QAAzB,MAAsCA,QAA1C,EAAoD;cAChDC,UAAU,GAAG,IAAb;YACH;UACJ,CAXD,CAWE,MAAM,CACJ;UACH;;UAED,IAAIA,UAAJ,EAAgB;YACZ,IAAI,OAAO7B,gBAAP,KAA4B,UAAhC,EAA4C;cACxC,MAAMA,gBAAgB,CAAC;gBAAEW,KAAK,EAAE;kBAAEyB,IAAI,EAAEzC,IAAR;kBAAc+B,QAAQ,EAAED;gBAAxB;cAAT,CAAD,CAAtB;YACH;UACJ,CAJD,MAIO;YACH,MAAMvB,YAAY,GAAGc,aAAa,CAACqB,IAAd,CAAmBC,IAAI,IAAIA,IAAI,CAACrB,OAAL,CAAasB,IAAb,CAAkBd,GAAlB,CAA3B,CAArB;;YACA,MAAMe,WAAW,GAAGC,aAAA,CAAKC,OAAL,CAAa/C,IAAb,CAApB;;YAEA,MAAM;cAAEgD,GAAF;cAAOC;YAAP,IAAkB,MAAM,IAAAC,kCAAA,EAAiB;cAC3ChD,MAD2C;cAE3C4B,GAF2C;cAG3CxB,GAH2C;cAI3C2B,QAJ2C;cAK3CY,WAL2C;cAM3CtC,YAAY,EAAEA,YAAY,GAAGA,YAAY,CAACgB,KAAhB,GAAwB4B;YANP,CAAjB,CAA9B;YASA,MAAMvD,IAAqC,+DACpCqD,MADoC;cAEvC,gBAAgBJ,WAAW,IAAI,EAFQ;cAGvC,uBAAuBZ,QAHgB;cAIvChD,IAAI,EAAEQ,WAAA,CAAG2D,YAAH,CAAgBpD,IAAhB;YAJiC,EAA3C;;YAOA,IAAIO,YAAJ,EAAkB;cACdX,IAAI,CAAC,eAAD,CAAJ,GAAwBW,YAAY,CAACgB,KAArC;YACH;;YAED,MAAM8B,QAAQ,GAAG,IAAIC,iBAAJ,EAAjB;YACAC,MAAM,CAACC,IAAP,CAAY5D,IAAZ,EAAkB6D,OAAlB,CAA0B3B,GAAG,IAAI;cAC7BuB,QAAQ,CAACK,MAAT,CAAgB5B,GAAhB,EAAqBlC,IAAI,CAACkC,GAAD,CAAzB;YACH,CAFD;YAIA,MAAM6B,GAAG,GAAG,MAAM,IAAAC,kBAAA,EAAMZ,GAAN,EAAW;cACzBa,MAAM,EAAE,MADiB;cAEzBC,IAAI,EAAET;YAFmB,CAAX,CAAlB;;YAKA,IAAIM,GAAG,CAACI,MAAJ,GAAa,GAAjB,EAAsB;cAClB,MAAM,IAAIhD,KAAJ,CAAW,GAAE4C,GAAG,CAACK,UAAW,KAAI,MAAML,GAAG,CAACM,IAAJ,EAAW,EAAjD,CAAN;YACH;;YAED,IAAI,OAAO9D,mBAAP,KAA+B,UAAnC,EAA+C;cAC3C,MAAMA,mBAAmB,CAAC;gBAAEa,KAAK,EAAE;kBAAEyB,IAAI,EAAEzC,IAAR;kBAAc+B,QAAQ,EAAED;gBAAxB;cAAT,CAAD,CAAzB;YACH;UACJ;;UACDvC,OAAO;QACV,CAnED,CAmEE,OAAO2E,CAAP,EAAU;UACR,IAAI,OAAO9D,iBAAP,KAA6B,UAAjC,EAA6C;YACzC,MAAMA,iBAAiB,CAAC;cACpBY,KAAK,EAAE;gBAAEyB,IAAI,EAAEzC,IAAR;gBAAc+B,QAAQ,EAAED;cAAxB,CADa;cAEpBqC,KAAK,EAAED;YAFa,CAAD,CAAvB;UAIH;;UACD3E,OAAO;QACV;MACJ,CA/ED,CADJ;IAkFH;;IAED,MAAMD,OAAO,CAAC8E,GAAR,CAAYxC,QAAZ,CAAN;EACH;AACJ,CA5HM"}