@automattic/vip 2.22.0 → 2.23.0

package/dist/lib/client-file-uploader.js

@@ -3,52 +3,38 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.getFileMeta = getFileMeta;
-exports.uploadImportSqlFileToS3 = uploadImportSqlFileToS3;
-exports.uploadUsingPutObject = uploadUsingPutObject;
-exports.uploadUsingMultipart = uploadUsingMultipart;
-exports.getSignedUploadRequestData = getSignedUploadRequestData;
+exports.MULTIPART_THRESHOLD = exports.COMPRESS_THRESHOLD = void 0;
 exports.checkFileAccess = checkFileAccess;
-exports.getFileStats = getFileStats;
-exports.isFile = isFile;
-exports.getFileSize = getFileSize;
+exports.completeMultipartUpload = completeMultipartUpload;
 exports.detectCompressedMimeType = detectCompressedMimeType;
+exports.getFileMD5Hash = void 0;
+exports.getFileMeta = getFileMeta;
+exports.getFileSize = getFileSize;
+exports.getFileStats = getFileStats;
 exports.getPartBoundaries = getPartBoundaries;
-exports.uploadParts = uploadParts;
+exports.getSignedUploadRequestData = getSignedUploadRequestData;
+exports.gzipFile = exports.getWorkingTempDir = void 0;
+exports.isFile = isFile;
+exports.uploadImportSqlFileToS3 = uploadImportSqlFileToS3;
 exports.uploadPart = uploadPart;
-exports.completeMultipartUpload = completeMultipartUpload;
-exports.gzipFile = exports.getFileMD5Hash = exports.getWorkingTempDir = exports.MULTIPART_THRESHOLD = exports.COMPRESS_THRESHOLD = void 0;
-
+exports.uploadParts = uploadParts;
+exports.uploadUsingMultipart = uploadUsingMultipart;
+exports.uploadUsingPutObject = uploadUsingPutObject;
 var _fs = _interopRequireWildcard(require("fs"));
-
 var _os = _interopRequireDefault(require("os"));
-
 var _path = _interopRequireDefault(require("path"));
-
 var _nodeFetch = _interopRequireDefault(require("node-fetch"));
-
 var _chalk = _interopRequireDefault(require("chalk"));
-
 var _zlib = require("zlib");
-
 var _crypto = require("crypto");
-
 var _stream = require("stream");
-
 var _xml2js = require("xml2js");
-
 var _debug = _interopRequireDefault(require("debug"));
-
 var _http = _interopRequireDefault(require("./api/http"));
-
 var _fileSize = require("./constants/file-size");
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
 function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
-
 function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
-
 /**
  * 
  * @format
@@ -61,48 +47,45 @@ function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj &&
 /**
  * Internal dependencies
  */
-const debug = (0, _debug.default)('vip:lib/client-file-uploader'); // Files smaller than COMPRESS_THRESHOLD will not be compressed before upload
 
-const COMPRESS_THRESHOLD = 16 * _fileSize.MB_IN_BYTES; // Files smaller than MULTIPART_THRESHOLD will use `PutObject` vs Multipart Uploads
+const debug = (0, _debug.default)('vip:lib/client-file-uploader');
+
+// Files smaller than COMPRESS_THRESHOLD will not be compressed before upload
+const COMPRESS_THRESHOLD = 16 * _fileSize.MB_IN_BYTES;
 
+// Files smaller than MULTIPART_THRESHOLD will use `PutObject` vs Multipart Uploads
 exports.COMPRESS_THRESHOLD = COMPRESS_THRESHOLD;
-const MULTIPART_THRESHOLD = 32 * _fileSize.MB_IN_BYTES; // This is how big each part of a Multipart Upload is (except the last / remainder)
+const MULTIPART_THRESHOLD = 32 * _fileSize.MB_IN_BYTES;
 
+// This is how big each part of a Multipart Upload is (except the last / remainder)
 exports.MULTIPART_THRESHOLD = MULTIPART_THRESHOLD;
-const UPLOAD_PART_SIZE = 16 * _fileSize.MB_IN_BYTES; // How many parts will upload at the same time
+const UPLOAD_PART_SIZE = 16 * _fileSize.MB_IN_BYTES;
 
+// How many parts will upload at the same time
 const MAX_CONCURRENT_PART_UPLOADS = 5;
-
 const getWorkingTempDir = async () => new Promise((resolve, reject) => {
   _fs.default.mkdtemp(_path.default.join(_os.default.tmpdir(), 'vip-client-file-uploader'), (err, dir) => {
     if (err) {
       return reject(err);
     }
-
     resolve(dir);
   });
 });
-
 exports.getWorkingTempDir = getWorkingTempDir;
-
 const getFileMD5Hash = async fileName => new Promise((resolve, reject) => _fs.default.createReadStream(fileName).pipe((0, _crypto.createHash)('md5').setEncoding('hex')).on('finish', function () {
   resolve(this.read());
 }).on('error', error => reject(`could not generate file hash: ${error}`)));
-
 exports.getFileMD5Hash = getFileMD5Hash;
-
 const gzipFile = async (uncompressedFileName, compressedFileName) => new Promise((resolve, reject) => _fs.default.createReadStream(uncompressedFileName).pipe((0, _zlib.createGzip)()).pipe(_fs.default.createWriteStream(compressedFileName)).on('finish', resolve).on('error', error => reject(`could not compress file: ${error}`)));
-
 exports.gzipFile = gzipFile;
-
 async function getFileMeta(fileName) {
   return new Promise(async resolve => {
     const fileSize = await getFileSize(fileName);
+    const basename = _path.default.basename(fileName);
+    // TODO Validate File basename...  encodeURIComponent, maybe...?
 
-    const basename = _path.default.basename(fileName); // TODO Validate File basename...  encodeURIComponent, maybe...?
-
-
-    const mimeType = await detectCompressedMimeType(fileName); // TODO Only allow a subset of Mime Types...?
+    const mimeType = await detectCompressedMimeType(fileName);
+    // TODO Only allow a subset of Mime Types...?
 
     const isCompressed = ['application/zip', 'application/gzip'].includes(mimeType);
     resolve({
@@ -113,7 +96,6 @@ async function getFileMeta(fileName) {
     });
   });
 }
-
 async function uploadImportSqlFileToS3({
   app,
   env,
@@ -121,14 +103,14 @@ async function uploadImportSqlFileToS3({
   progressCallback
 }) {
   let tmpDir;
-
   try {
     tmpDir = await getWorkingTempDir();
   } catch (err) {
     throw `Unable to create temporary working directory: ${err}`;
   }
+  debug(`File ${_chalk.default.cyan(fileMeta.basename)} is ~ ${Math.floor(fileMeta.fileSize / _fileSize.MB_IN_BYTES)} MB\n`);
 
-  debug(`File ${_chalk.default.cyan(fileMeta.basename)} is ~ ${Math.floor(fileMeta.fileSize / _fileSize.MB_IN_BYTES)} MB\n`); // TODO Compression will probably fail over a certain file size... break into pieces...?
+  // TODO Compression will probably fail over a certain file size... break into pieces...?
   // TODO if needed add a flag to bypass auto-compression
 
   if (!fileMeta.isCompressed && fileMeta.fileSize >= COMPRESS_THRESHOLD) {
@@ -146,7 +128,6 @@ async function uploadImportSqlFileToS3({
     const calculation = `${(fewerBytes / _fileSize.MB_IN_BYTES).toFixed(2)}MB (${Math.floor(100 * fewerBytes / uncompressedFileSize)}%)`;
     debug(`** Compression resulted in a ${calculation} smaller file 📦 **\n`);
   }
-
   debug('Calculating file md5 checksum...');
   const md5 = await getFileMD5Hash(fileMeta.fileName);
   debug(`Calculated file md5 checksum: ${md5}\n`);
@@ -167,7 +148,6 @@ async function uploadImportSqlFileToS3({
     result
   };
 }
-
 async function uploadUsingPutObject({
   app,
   env,
@@ -187,43 +167,41 @@ async function uploadUsingPutObject({
     action: 'PutObject'
   });
   const fetchOptions = presignedRequest.options;
-  fetchOptions.headers = { ...fetchOptions.headers,
+  fetchOptions.headers = {
+    ...fetchOptions.headers,
     'Content-Length': `${fileSize}` // This has to be a string
-
   };
+
   let readBytes = 0;
   const progressPassThrough = new _stream.PassThrough();
   progressPassThrough.on('data', data => {
     readBytes += data.length;
     const percentage = Math.floor(100 * readBytes / fileSize) + '%';
     debug(percentage);
-
     if (typeof progressCallback === 'function') {
       progressCallback(percentage);
     }
   });
-  const response = await (0, _nodeFetch.default)(presignedRequest.url, { ...fetchOptions,
+  const response = await (0, _nodeFetch.default)(presignedRequest.url, {
+    ...fetchOptions,
     body: fileContent ? fileContent : _fs.default.createReadStream(fileName).pipe(progressPassThrough)
   });
-
   if (response.status === 200) {
     return 'ok';
   }
+  const result = await response.text();
 
-  const result = await response.text(); // TODO is any additional hardening needed here?
-
+  // TODO is any additional hardening needed here?
   const parser = new _xml2js.Parser({
     explicitArray: false,
     ignoreAttrs: true
   });
   let parsedResponse;
-
   try {
     parsedResponse = await parser.parseStringPromise(result);
   } catch (err) {
     throw `Invalid response from cloud service. ${err}`;
   }
-
   const {
     Code,
     Message
@@ -233,7 +211,6 @@ async function uploadUsingPutObject({
     Message
   })}`;
 }
-
 async function uploadUsingMultipart({
   app,
   env,
@@ -251,14 +228,14 @@ async function uploadUsingMultipart({
     action: 'CreateMultipartUpload'
   });
   const multipartUploadResponse = await (0, _nodeFetch.default)(presignedCreateMultipartUpload.url, presignedCreateMultipartUpload.options);
-  const multipartUploadResult = await multipartUploadResponse.text(); // TODO is any hardening needed here?
+  const multipartUploadResult = await multipartUploadResponse.text();
 
+  // TODO is any hardening needed here?
   const parser = new _xml2js.Parser({
     explicitArray: false,
     ignoreAttrs: true
   });
   const parsedResponse = await parser.parseStringPromise(multipartUploadResult);
-
   if (parsedResponse.Error) {
     const {
       Code,
@@ -269,11 +246,9 @@ async function uploadUsingMultipart({
       Message
     })}`;
   }
-
   if (!parsedResponse && parsedResponse.InitiateMultipartUploadResult && parsedResponse.InitiateMultipartUploadResult.UploadId) {
     throw `Unable to get Upload ID from cloud storage. Error: ${multipartUploadResult}`;
   }
-
   const uploadId = parsedResponse.InitiateMultipartUploadResult.UploadId;
   debug({
     uploadId
@@ -298,7 +273,6 @@ async function uploadUsingMultipart({
     etagResults
   });
 }
-
 async function getSignedUploadRequestData({
   action,
   appId,
@@ -320,22 +294,17 @@ async function getSignedUploadRequestData({
       uploadId
     }
   });
-
   if (response.status !== 200) {
     throw (await response.text()) || response.statusText;
   }
-
   return response.json();
 }
-
 async function checkFileAccess(fileName) {
   return _fs.default.promises.access(fileName, _fs.default.R_OK);
 }
-
 async function getFileStats(fileName) {
   return _fs.default.promises.stat(fileName);
 }
-
 async function isFile(fileName) {
   try {
     const stats = await getFileStats(fileName);
@@ -345,12 +314,10 @@ async function isFile(fileName) {
     return false;
   }
 }
-
 async function getFileSize(fileName) {
   const stats = await getFileStats(fileName);
   return stats.size;
 }
-
 async function detectCompressedMimeType(fileName) {
   const ZIP_MAGIC_NUMBER = '504b0304';
   const GZ_MAGIC_NUMBER = '1f8b';
@@ -366,21 +333,17 @@ async function detectCompressedMimeType(fileName) {
       if (ZIP_MAGIC_NUMBER === fileHeader.slice(0, ZIP_MAGIC_NUMBER.length)) {
         return resolve('application/zip');
       }
-
       if (GZ_MAGIC_NUMBER === fileHeader.slice(0, GZ_MAGIC_NUMBER.length)) {
         return resolve('application/gzip');
       }
-
       resolve();
     });
   });
 }
-
 function getPartBoundaries(fileSize) {
   if (fileSize < 1) {
     throw 'fileSize must be greater than zero';
   }
-
   const numParts = Math.ceil(fileSize / UPLOAD_PART_SIZE);
   return new Array(numParts).fill(undefined).map((_numPart, index) => {
     const start = index * UPLOAD_PART_SIZE;
@@ -395,7 +358,6 @@ function getPartBoundaries(fileSize) {
     };
   });
 }
-
 async function uploadParts({
   app,
   env,
@@ -407,7 +369,6 @@ async function uploadParts({
   let uploadsInProgress = 0;
   let totalBytesRead = 0;
   const partPercentages = new Array(parts.length).fill(0);
-
   const readyForPartUpload = () => new Promise(resolve => {
     const canDoInterval = setInterval(() => {
       if (uploadsInProgress < MAX_CONCURRENT_PART_UPLOADS) {
@@ -417,14 +378,11 @@ async function uploadParts({
       }
     }, 300);
   });
-
   const updateProgress = () => {
     const percentage = Math.floor(100 * totalBytesRead / fileMeta.fileSize) + '%';
-
     if (typeof progressCallback === 'function') {
       progressCallback(percentage);
     }
-
     debug(partPercentages.map((partPercentage, index) => {
       const {
         partSize
@@ -432,7 +390,6 @@ async function uploadParts({
       return `Part # ${index}: ${partPercentage}% of ${(partSize / _fileSize.MB_IN_BYTES).toFixed(2)}MB`;
     }).join('\n') + `\n\nOverall Progress: ${percentage}% of ${(fileMeta.fileSize / _fileSize.MB_IN_BYTES).toFixed(2)}MB`);
   };
-
   const updateProgressInterval = setInterval(updateProgress, 500);
   const allDone = await Promise.all(parts.map(async part => {
     const {
@@ -462,7 +419,6 @@ async function uploadParts({
   updateProgress();
   return allDone;
 }
-
 async function uploadPart({
   app,
   env,
@@ -481,8 +437,8 @@ async function uploadPart({
     start
   } = part;
   const s3PartNumber = index + 1; // S3 multipart is indexed from 1
-  // TODO: handle failures / retries, etc.
 
+  // TODO: handle failures / retries, etc.
   const doUpload = async () => {
     // Get the signed request data from Parker
     const partUploadRequestData = await getSignedUploadRequestData({
@@ -494,37 +450,35 @@ async function uploadPart({
       uploadId
     });
     const fetchOptions = partUploadRequestData.options;
-    fetchOptions.headers = { ...fetchOptions.headers,
+    fetchOptions.headers = {
+      ...fetchOptions.headers,
       'Content-Length': `${partSize}` // This has to be a string
-
       /**
        * TODO? 'Content-MD5': Buffer.from( ... ).toString( 'base64' ),
        * Content-MD5 has to be base64 encoded.
        * It's the hash of the entire request object & has to be included in the signature,
        *   ...so it may not be feasible to include with presigned requests.
        */
-
     };
+
     fetchOptions.body = _fs.default.createReadStream(fileName, {
       start,
       end
     }).pipe(progressPassThrough);
     const fetchResponse = await (0, _nodeFetch.default)(partUploadRequestData.url, fetchOptions);
-
     if (fetchResponse.status === 200) {
       const responseHeaders = fetchResponse.headers.raw();
       const [etag] = responseHeaders.etag;
       return JSON.parse(etag);
     }
+    const result = await fetchResponse.text();
 
-    const result = await fetchResponse.text(); // TODO is any hardening needed here?
-
+    // TODO is any hardening needed here?
     const parser = new _xml2js.Parser({
       explicitArray: false,
       ignoreAttrs: true
     });
     const parsed = await parser.parseStringPromise(result);
-
     if (parsed.Error) {
       const {
         Code,
@@ -535,16 +489,13 @@ async function uploadPart({
         Message
       })}`;
     }
-
     return parsed;
   };
-
   return {
     ETag: await doUpload(),
     PartNumber: s3PartNumber
   };
 }
-
 async function completeMultipartUpload({
   app,
   env,
@@ -561,10 +512,10 @@ async function completeMultipartUpload({
     etagResults
   });
   const completeMultipartUploadResponse = await (0, _nodeFetch.default)(completeMultipartUploadRequestData.url, completeMultipartUploadRequestData.options);
-
   if (completeMultipartUploadResponse.status !== 200) {
     throw await completeMultipartUploadResponse.text();
   }
+
   /**
    * Processing of a Complete Multipart Upload request could take several minutes to complete.
    * After Amazon S3 begins processing the request, it sends an HTTP response header that specifies a 200 OK response.
@@ -575,15 +526,12 @@ async function completeMultipartUpload({
    *
    * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
    */
-
-
   const result = await completeMultipartUploadResponse.text();
   const parser = new _xml2js.Parser({
     explicitArray: false,
     ignoreAttrs: true
   });
   const parsed = await parser.parseStringPromise(result);
-
   if (parsed.Error) {
     const {
       Code,
@@ -594,6 +542,5 @@ async function completeMultipartUpload({
       Message
     })}`;
   }
-
   return parsed;
 }