@alwaysmeticulous/remote-replay-launcher 2.255.0 → 2.255.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/asset-upload-utils.js +4 -3
- package/dist/asset-upload-utils.js.map +1 -1
- package/dist/execute-remote-test-run.js +2 -2
- package/dist/execute-remote-test-run.js.map +1 -1
- package/dist/upload-utils/__tests__/{multipart-zip-uploader.spec.js → multipart-compressing-uploader.spec.js} +8 -8
- package/dist/upload-utils/__tests__/multipart-compressing-uploader.spec.js.map +1 -0
- package/dist/upload-utils/{multipart-zip-uploader.d.ts → multipart-compressing-uploader.d.ts} +4 -9
- package/dist/upload-utils/multipart-compressing-uploader.js +89 -0
- package/dist/upload-utils/multipart-compressing-uploader.js.map +1 -0
- package/package.json +6 -6
- package/dist/upload-utils/__tests__/multipart-zip-uploader.spec.js.map +0 -1
- package/dist/upload-utils/multipart-zip-uploader.js +0 -170
- package/dist/upload-utils/multipart-zip-uploader.js.map +0 -1
- /package/dist/upload-utils/__tests__/{multipart-zip-uploader.spec.d.ts → multipart-compressing-uploader.spec.d.ts} +0 -0
|
@@ -33,7 +33,7 @@ const project_deployments_api_1 = require("@alwaysmeticulous/client/dist/api/pro
|
|
|
33
33
|
const common_1 = require("@alwaysmeticulous/common");
|
|
34
34
|
const Sentry = __importStar(require("@sentry/node"));
|
|
35
35
|
const poll_for_base_test_run_1 = require("./poll-for-base-test-run");
|
|
36
|
-
const
|
|
36
|
+
const multipart_compressing_uploader_1 = require("./upload-utils/multipart-compressing-uploader");
|
|
37
37
|
/**
|
|
38
38
|
* Uploads assets from a directory and returns the upload ID and client for further operations
|
|
39
39
|
*/
|
|
@@ -76,6 +76,7 @@ const completeUploadAndWaitForBase = async ({ client, uploadId, commitSha, waitF
|
|
|
76
76
|
mustHaveBase: waitForBase,
|
|
77
77
|
rewrites,
|
|
78
78
|
createDeployment,
|
|
79
|
+
archiveType: multipart_compressing_uploader_1.UPLOAD_ARCHIVE_FILE_FORMAT,
|
|
79
80
|
...(multipartUploadInfo ? { multipartUploadInfo } : {}),
|
|
80
81
|
};
|
|
81
82
|
const initialResult = await (0, client_1.completeAssetUpload)(completeAssetUploadArgs);
|
|
@@ -108,9 +109,9 @@ const completeUploadAndWaitForBase = async ({ client, uploadId, commitSha, waitF
|
|
|
108
109
|
};
|
|
109
110
|
const uploadAssetsStreaming = async ({ client, folderPath, commitSha, waitForBase = false, rewrites = [], createDeployment = true, }) => {
|
|
110
111
|
const logger = (0, common_1.initLogger)();
|
|
111
|
-
const { uploadId, awsUploadId, uploadPartUrls, uploadChunkSize } = await (0, client_1.requestMultipartAssetUpload)({ client });
|
|
112
|
+
const { uploadId, awsUploadId, uploadPartUrls, uploadChunkSize } = await (0, client_1.requestMultipartAssetUpload)({ client, archiveType: multipart_compressing_uploader_1.UPLOAD_ARCHIVE_FILE_FORMAT });
|
|
112
113
|
logger.info(`Starting streaming upload for deployment ${uploadId}`);
|
|
113
|
-
const uploader = new
|
|
114
|
+
const uploader = new multipart_compressing_uploader_1.MultipartCompressingUploader({
|
|
114
115
|
folderPath,
|
|
115
116
|
uploadPartUrls,
|
|
116
117
|
uploadChunkSize,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"asset-upload-utils.js","sourceRoot":"","sources":["../src/asset-upload-utils.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2BAAkD;AAClD,0CAA2C;AAE3C,iCAAgD;AAChD,+BAAqC;AAErC,qDASkC;AAClC,uGAAmG;AACnG,qDAAsD;AACtD,qDAAuC;AACvC,qEAAiE;AACjE,
|
|
1
|
+
{"version":3,"file":"asset-upload-utils.js","sourceRoot":"","sources":["../src/asset-upload-utils.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2BAAkD;AAClD,0CAA2C;AAE3C,iCAAgD;AAChD,+BAAqC;AAErC,qDASkC;AAClC,uGAAmG;AACnG,qDAAsD;AACtD,qDAAuC;AACvC,qEAAiE;AACjE,kGAAyH;AAgBzH;;GAEG;AACI,MAAM,YAAY,GAAG,KAAK,EAC/B,IAGC,EAC4B,EAAE;IAC/B,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IAC5B,MAAM,EAAE,YAAY,EAAE,iBAAiB,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAEtE,MAAM,oBAAoB,GAAG,IAAA,cAAO,EAAC,YAAY,CAAC,CAAC;IACnD,IAAI,CAAC,IAAA,eAAU,EAAC,oBAAoB,CAAC,EAAE;QACrC,MAAM,IAAI,KAAK,CAAC,6BAA6B,oBAAoB,EAAE,CAAC,CAAC;KACtE;IAED,IAAI,iBAAiB,EAAE;QACrB,MAAM,aAAa,GAAG,IAAA,WAAI,EAAC,oBAAoB,EAAE,YAAY,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAA,eAAU,EAAC,aAAa,CAAC,EAAE;YAC9B,MAAM,CAAC,IAAI,CACT,sDAAsD,oBAAoB,KAAK;gBAC/E,0IAA0I;gBAC1I,oFAAoF;gBACpF,uGAAuG,CACxG,CAAC;SACH;KACF;IAED,MAAM,QAAQ,GAAG,IAAA,oBAAW,EAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,EAAE;QACb,MAAM,CAAC,KAAK,CACV,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,MAAM,MAAM,GAAG,IAAA,qBAAY,EAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE1C,OAAO,qBAAqB,CAAC;QAC3B,GAAG,IAAI;QACP,MAAM;QACN,UAAU,EAAE,oBAAoB;KACjC,CAAC,CAAC;AACL,CAAC,CAAC;AAzCW,QAAA,YAAY,gBAyCvB;AAEF,MAAM,4BAA4B,GAAG,KAAK,EAAE,EAC1C,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,EACX,QAAQ,EACR,gBAAgB,EAChB,mBAAmB,GASpB,EAGE,EAAE;;IACH,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IAE5B,MAAM,uBAAuB,GAAG;QAC9B,MAAM;QACN,QAAQ;QACR,SAAS;QACT,YAAY,EAAE,WAAW;QACzB,QAAQ;QACR,gBAAgB;QAChB,WAAW,EAAE,2DAA0B;QACvC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxD,CAAC;IAEF,MAAM,aAAa,GAAG,MAAM,IAAA,4BAAmB,EAAC,uBAAuB,CAAC,CAAC;IACzE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,MAAM,IAAA,8CAAqB,EAAC;QACrE,aAAa,EAAE;YACb,OAAO,EAAE,MAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,OAAO,mCAAI,IAAI;YACvC,YAAY,EAAE,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,YAAY;YACzC,OAAO,EAAE,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,OAAO;SAChC;QACD,OAAO,EAAE,GAAG,EAAE,CAAC,IAAA,gDAAsB,EAAC,uBAAuB,CAAC;QAC9D,UAAU,EAAE,GAAG,EAAE,CACf,IAAA,gDAAsB,EAAC;YACrB,GAAG,uBAAuB;YAC1B,YAAY,EAAE,KAAK;SACpB,CAAC;KACL,CAAC,CAAC;IAEH,MAAM,CAAC,cAAc,CAAC,sCAAsC,EAAE;QAC5D,KAAK,EAAE,OAAO;QACd,KAAK,EAAE;YACL,QAAQ,EAAE,QAAQ;YAClB,SAAS,EAAE,SAAS;YACpB,SAAS,EAAE,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,EAAE;YACtB,YAAY,EAAE,YAAY;SAC3B;KACF,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,qBAAqB,CAAC,CAAC;IAEhE,OAAO;QACL,OAAO,EAAE,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAI;QACxB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,KAAK,EAAE,EACnC,MAAM,EACN,UAAU,EACV,SAAS,EACT,WAAW,GAAG,KAAK,EACnB,QAAQ,GAAG,EAAE,EACb,gBAAgB,GAAG,IAAI,GAIxB,EAA+B,EAAE;IAChC,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IAE5B,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,GAC9D,MAAM,IAAA,oCAA2B,EAAC,EAAE,MAAM,EAAE,WAAW,EAAE,2DAA0B,EAAE,CAAC,CAAC;IAEzF,MAAM,CAAC,IAAI,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;IAEpE,MAAM,QAAQ,GAAG,IAAI,6DAA4B,CAAC;QAChD,UAAU;QACV,cAAc;QACd,eAAe;QACf,WAAW;QACX,QAAQ;QACR,MAAM;QACN,uBAAuB;KACxB,CAAC,CAAC;IACH,MAAM,mBAAmB,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IAErD,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,wBAAwB,CAAC,CAAC;IAEnE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,4BAA4B,CAAC;QAC9D,MAAM;QACN,QAAQ;QACR,SAAS;QACT,WAAW;QACX,QAAQ;QACR,gBAAgB;QAChB,mBAAmB;KACpB,CAAC,CAAC;IAEH,OAAO;QACL,QAAQ;QACR,OAAO;QACP,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,KAAK,EACnC,SAAiB,EACjB,MAAc,EACG,EAAE;IACnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,IAAA,eAAY,EACtB,SAAS,EACT;YACE,KAAK,EAAE,IAAA,sBAAa,GAAE;YACtB,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,gBAAgB,EAAE,MAAM,CAAC,MAAM;aAChC;SACF,EACD,CAAC,QAAyB,EAAE,EAAE;YAC5B,IAAI,YAAY,GAAG,EAAE,CAAC;YAEtB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC5B,YAAY,IAAI,KAAK,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE;oBAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBACtC,IAAI,CAAC,IAAI,EAAE;wBACT,MAAM,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;wBAC9C,OAAO;qBACR;oBACD,OAAO,CAAC,IAAI,CAAC,CAAC;iBACf;qBAAM;oBACL,MAAM,YAAY,GAAG,kCAAkC,QAAQ,CAAC,UAAU,iBAAiB,YAAY,EAAE,CAAC;oBAC1G,MAAM,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;iBACjC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CACF,CAAC;QAEF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YACxB,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClB,GAAG,CAAC,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEK,MAAM,mBAAmB,GAAG,KAAK,EAAE,EACxC,QAAQ,EAAE,SAAS,EACnB,OAAO,EACP,SAAS,EACT,WAAW,GAAG,KAAK,EACnB,QAAQ,GAAG,EAAE,EACb,gBAAgB,GAAG,IAAI,EACvB,iBAAiB,GAAG,KAAK,GAI1B,EAA+B,EAAE;IAChC,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IAE5B,MAAM,QAAQ,GAAG,IAAA,oBAAW,EAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,EAAE;QACb,MAAM,CAAC,KAAK,CACV,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,MAAM,MAAM,GAAG,IAAA,qBAAY,EAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE1C,IAAI;QACF,MAAM,SAAS,GAAG,MAAM,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC;QAChC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,MAAM,IAAA,2BAAkB,EAAC;YACvD,MAAM;YACN,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;QACH,MAAM,qBAAqB,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC1D,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,wBAAwB,CAAC,CAAC;QAEnE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,4BAA4B,CAAC;YAC9D,MAAM;YACN,QAAQ;YACR,SAAS;YACT,WAAW;YACX,QAAQ;YACR,gBAAgB;SACjB,CAAC,CAAC;QAEH,OAAO;YACL,QAAQ;YACR,OAAO;YACP,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChC,CAAC;KACH;YAAS;QACR,IAAI,iBAAiB,EAAE;YACrB,IAAI;gBACF,MAAM,IAAA,iBAAM,EAAC,OAAO,CAAC,CAAC;aACvB;YAAC,OAAO,KAAK,EAAE;gBACd,MAAM,CAAC,IAAI,CAAC,mCAAmC,OAAO,KAAK,KAAK,EAAE,CAAC,CAAC;aACrE;SACF;KACF;AACH,CAAC,CAAC;AAzDW,QAAA,mBAAmB,uBAyD9B;AAEF,MAAM,qBAAqB,GAAG,KAAK,EACjC,QAAgB,EAChB,SAAiB,EACjB,gBAAwB,EACT,EAAE;IACjB,MAAM,UAAU,GAAG,IAAA,qBAAgB,EAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,MAAM,IAAA,eAAI,EAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC;IAChC,IAAI,QAAQ,KAAK,gBAAgB,EAAE;QACjC,MAAM,IAAI,KAAK,CACb,gCAAgC,gBAAgB,eAAe,QAAQ,QAAQ,CAChF,CAAC;KACH;IACD,MAAM,CAAC,IAAI,CAAC,gCAAgC,QAAQ,YAAY,CAAC,CAAC;IAElE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,IAAA,eAAY,EACtB,SAAS,EACT;YACE,KAAK,EAAE,IAAA,sBAAa,GAAE;YACtB,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,gBAAgB,EAAE,QAAQ;gBAC1B,cAAc,EAAE,iBAAiB;aAClC;SACF,EACD,CAAC,QAAyB,EAAE,EAAE;YAC5B,IAAI,YAAY,GAAG,EAAE,CAAC;YAEtB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC5B,YAAY,IAAI,KAAK,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE;oBAC/B,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;oBACvD,OAAO,EAAE,CAAC;iBACX;qBAAM;oBACL,MAAM,YAAY,GAAG,yCAAyC,SAAS,YAAY,QAAQ,CAAC,UAAU,iBAAiB,YAAY,EAAE,CAAC;oBACtI,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;oBAC3B,MAAM,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;iBACjC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CACF,CAAC;QAEF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YACxB,MAAM,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACvD,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC/B,MAAM,CAAC,KAAK,CAAC,sBAAsB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACpD,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC"}
|
|
@@ -6,7 +6,7 @@ const common_1 = require("@alwaysmeticulous/common");
|
|
|
6
6
|
const tunnels_client_1 = require("@alwaysmeticulous/tunnels-client");
|
|
7
7
|
const asset_upload_utils_1 = require("./asset-upload-utils");
|
|
8
8
|
const resource_tracker_1 = require("./resource-tracker");
|
|
9
|
-
const
|
|
9
|
+
const multipart_compressing_uploader_1 = require("./upload-utils/multipart-compressing-uploader");
|
|
10
10
|
const url_utils_1 = require("./url.utils");
|
|
11
11
|
const POLL_LOCK_INTERVAL_MS = 5000; // 5 seconds
|
|
12
12
|
const PROGRESS_UPDATE_INTERVAL_MS = 5000; // 5 seconds
|
|
@@ -60,7 +60,7 @@ const executeRemoteTestRun = async ({ apiToken: apiToken_, appUrl, commitSha, se
|
|
|
60
60
|
companionAssetsInfo = {
|
|
61
61
|
deploymentUploadId: uploadId,
|
|
62
62
|
regex,
|
|
63
|
-
archiveType:
|
|
63
|
+
archiveType: multipart_compressing_uploader_1.UPLOAD_ARCHIVE_FILE_FORMAT,
|
|
64
64
|
};
|
|
65
65
|
logger.info(`Companion assets uploaded with ID: ${uploadId}`);
|
|
66
66
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execute-remote-test-run.js","sourceRoot":"","sources":["../src/execute-remote-test-run.ts"],"names":[],"mappings":";;;AACA,qDAQkC;AAClC,qDAA6D;AAC7D,qEAA+D;AAC/D,6DAAyE;AACzE,yDAAqD;AAKrD,
|
|
1
|
+
{"version":3,"file":"execute-remote-test-run.js","sourceRoot":"","sources":["../src/execute-remote-test-run.ts"],"names":[],"mappings":";;;AACA,qDAQkC;AAClC,qDAA6D;AAC7D,qEAA+D;AAC/D,6DAAyE;AACzE,yDAAqD;AAKrD,kGAA2F;AAC3F,2CAAsC;AAItC,MAAM,qBAAqB,GAAG,IAAK,CAAC,CAAC,YAAY;AACjD,MAAM,2BAA2B,GAAG,IAAK,CAAC,CAAC,YAAY;AACvD,MAAM,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAK,CAAC,CAAC,YAAY;AAElD,MAAM,oBAAoB,GAAG,KAAK,EAAE,EACzC,QAAQ,EAAE,SAAS,EACnB,MAAM,EACN,SAAS,EACT,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,qBAAqB,EACrB,WAAW,EACX,UAAU,EACV,4BAA4B,EAC5B,eAAe,EACf,gBAAgB,GAAG,KAAK,EACxB,YAAY,GAAG,KAAK,EACpB,uBAAuB,GAAG,KAAK,EAC/B,cAAc,GAAG,KAAK,EACtB,gBAAgB,EAChB,mBAAmB,GAAG,KAAK,EAC3B,WAAW,GAAG,KAAK,GACS,EAAuC,EAAE;;IACrE,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IAE5B,MAAM,QAAQ,GAAG,IAAA,oBAAW,EAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,EAAE;QACb,MAAM,CAAC,KAAK,CACV,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,MAAM,MAAM,GAAG,IAAA,qBAAY,EAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE1C,IAAI,GAAQ,CAAC;IACb,IAAI;QACF,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QACtB,6DAA6D;KAC9D;IAAC,OAAO,MAAM,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,oBAAoB,MAAM,EAAE,CAAC,CAAC;KAC/C;IAED,MAAM,IAAI,GAAG,IAAA,mBAAO,EAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,IAAI,KAAK,CAAC,CAAC,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,EAAE,CAAC,CAAC;KACpD;IAED,IAAI,mBAAmB,GAAoC,SAAS,CAAC;IACrE,IAAI,eAAe,EAAE;QACnB,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,mCAAmC,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,GAAG,EAAE,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG;YACX,QAAQ;YACR,SAAS;YACT,WAAW,EAAE,KAAK;YAClB,QAAQ,EAAE,EAAE;YACZ,gBAAgB,EAAE,KAAK;YACvB,iBAAiB,EAAE,KAAK;SACzB,CAAC;QACF,MAAM,MAAM,GAAG,MAAM;YACnB,CAAC,CAAC,MAAM,IAAA,iCAAY,EAAC;gBACjB,GAAG,IAAI;gBACP,YAAY,EAAE,MAAM;gBACpB,iBAAiB,EAAE,KAAK;aACzB,CAAC;YACJ,CAAC,CAAC,GAAG;gBACH,CAAC,CAAC,MAAM,IAAA,wCAAmB,EAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;gBACtD,CAAC,CAAC,SAAS,CAAC;QAChB,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;SAClE;QACD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAC5B,mBAAmB,GAAG;YACpB,kBAAkB,EAAE,QAAQ;YAC5B,KAAK;YACL,WAAW,EAAE,2DAA0B;SACxC,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;KAC/D;IAED,MAAM,MAAM,GAAG,MAAM,IAAA,4BAAW,EAAC;QAC/B,MAAM;QACN,QAAQ;QACR,SAAS,EAAE,GAAG,CAAC,QAAQ;QACvB,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,IAAI;QACJ,UAAU,EAAE,GAAG,CAAC,QAAQ,KAAK,QAAQ;QACrC,gBAAgB;QAChB,YAAY;QACZ,uBAAuB;QACvB,cAAc;QACd,gBAAgB;QAChB,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAElC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE;QACrE,MAAM,IAAI,KAAK,CACb,wEAAwE,CACzE,CAAC;KACH;IAED,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAG;QAChB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;KAC5C,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,IAAA,mCAA0B,EAAC;QAChD,MAAM;QACN,OAAO,EAAE,SAAS;QAClB,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;QAC3C,WAAW;QACX,UAAU;QACV,WAAW;QACX,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,4BAA4B,CAAC,CAAC,CAAC,EAAE,4BAA4B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1E,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE;QACrB,MAAM,IAAI,KAAK,CAAC,GAAG,MAAA,QAAQ,CAAC,OAAO,mCAAI,0BAA0B,EAAE,CAAC,CAAC;KACtE;IAED,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,QAAQ,CAAC;IAC3C,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAG,OAAO,CAAC,CAAC;IAE5B,MAAM,gBAAgB,GAAG,IAAA,cAAK,GAAW,CAAC;IAE1C,IAAI,sBAAsB,GAA+B,SAAS,CAAC;IACnE,IAAI,wBAAwB,GAAuB,SAAS,CAAC;IAE7D,MAAM,kBAAkB,GAAG,CAAC,gBAAyB,EAAE,EAAE;QACvD,IACE,gBAAgB,CAAC,MAAM,KAAK,gBAAgB;YAC5C,CAAC,wBAAwB,KAAK,SAAS;gBACrC,IAAI,CAAC,GAAG,EAAE,GAAG,wBAAwB,GAAG,oBAAoB,CAAC,EAC/D;YACA,wFAAwF;YACxF,6FAA6F;YAC7F,oEAAoE;YACpE,IAAI,wBAAwB,KAAK,SAAS,EAAE;gBAC1C,wBAAwB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACtC,MAAM,CAAC,IAAI,CACT,qDACE,oBAAoB,GAAG,IACzB,qDAAqD,CACtD,CAAC;aACH;YACD,OAAO;SACR;QAED,IAAI,sBAAsB,EAAE;YAC1B,aAAa,CAAC,sBAAsB,CAAC,CAAC;SACvC;QAED,gBAAgB,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC7C,CAAC,CAAC;IAEF,sFAAsF;IACtF,MAAM,eAAe,GAAG,IAAI,kCAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7D,sBAAsB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;QAC9C,MAAM,cAAc,GAAG,MAAM,IAAA,mBAAU,EAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3E,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAG,cAAc,CAAC,CAAC;QAEnC,IAAI,CAAC,oCAA2B,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE;YAChE,kBAAkB,CAAC,cAAc,CAAC,CAAC;YAEnC,OAAO;SACR;aAAM,IAAI,wBAAwB,KAAK,SAAS,EAAE;YACjD,MAAM,CAAC,IAAI,CACT,uCAAuC,cAAc,CAAC,MAAM,GAAG,CAChE,CAAC;YACF,wBAAwB,GAAG,SAAS,CAAC;SACtC;QACD,MAAM,eAAe,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC,EAAE,2BAA2B,CAAC,CAAC;IAEhC,MAAM,gBAAgB,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC;IAExD,MAAM,cAAc,GAAG,IAAA,cAAK,GAAQ,CAAC;IACrC,IAAI,mBAAmB,GAA+B,SAAS,CAAC;IAChE,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;QAC/B,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAW,EAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QAC7D,IAAI,QAAQ,EAAE;YACZ,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,EAAI,CAAC;YACxB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC;SACd;QACD,cAAc,CAAC,OAAO,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,eAAe,GAAG,MAAM,aAAa,EAAE,CAAC;IAC9C,IAAI,CAAC,eAAe,EAAE;QACpB,mBAAmB,GAAG,WAAW,CAAC,aAAa,EAAE,qBAAqB,CAAC,CAAC;QACxE,MAAM,cAAc,CAAC,OAAO,CAAC;KAC9B;IACD,IAAI,mBAAmB,EAAE;QACvB,aAAa,CAAC,mBAAmB,CAAC,CAAC;KACpC;IAED,IAAI,qBAAqB,EAAE;QACzB,MAAM,qBAAqB,CAAC;KAC7B;IACD,MAAM,CAAC,KAAK,EAAE,CAAC;IAEf,OAAO;QACL,OAAO,EAAE,gBAAgB;KAC1B,CAAC;AACJ,CAAC,CAAC;AAlNW,QAAA,oBAAoB,wBAkN/B"}
|
|
@@ -4,7 +4,7 @@ const promises_1 = require("fs/promises");
|
|
|
4
4
|
const os_1 = require("os");
|
|
5
5
|
const path_1 = require("path");
|
|
6
6
|
const vitest_1 = require("vitest");
|
|
7
|
-
const
|
|
7
|
+
const multipart_compressing_uploader_1 = require("../multipart-compressing-uploader");
|
|
8
8
|
vitest_1.vi.mock("@alwaysmeticulous/client", async () => {
|
|
9
9
|
const actual = await vitest_1.vi.importActual("@alwaysmeticulous/client");
|
|
10
10
|
return {
|
|
@@ -14,11 +14,11 @@ vitest_1.vi.mock("@alwaysmeticulous/client", async () => {
|
|
|
14
14
|
.mockResolvedValue({ uploadPartUrl: "https://example.com/extra-part" }),
|
|
15
15
|
};
|
|
16
16
|
});
|
|
17
|
-
(0, vitest_1.describe)("
|
|
17
|
+
(0, vitest_1.describe)("MultipartCompressingUploader", () => {
|
|
18
18
|
let testDir;
|
|
19
19
|
const uploadedParts = [];
|
|
20
20
|
(0, vitest_1.beforeEach)(async () => {
|
|
21
|
-
testDir = (0, path_1.join)((0, os_1.tmpdir)(), `test-multipart-
|
|
21
|
+
testDir = (0, path_1.join)((0, os_1.tmpdir)(), `test-multipart-compressing-${Date.now()}`);
|
|
22
22
|
await (0, promises_1.mkdir)(testDir, { recursive: true });
|
|
23
23
|
uploadedParts.length = 0;
|
|
24
24
|
});
|
|
@@ -48,13 +48,13 @@ vitest_1.vi.mock("@alwaysmeticulous/client", async () => {
|
|
|
48
48
|
},
|
|
49
49
|
...overrides,
|
|
50
50
|
};
|
|
51
|
-
return new
|
|
51
|
+
return new multipart_compressing_uploader_1.MultipartCompressingUploader(args);
|
|
52
52
|
};
|
|
53
53
|
(0, vitest_1.it)("should create an instance with valid args", () => {
|
|
54
54
|
const uploader = createUploader();
|
|
55
|
-
(0, vitest_1.expect)(uploader).toBeInstanceOf(
|
|
55
|
+
(0, vitest_1.expect)(uploader).toBeInstanceOf(multipart_compressing_uploader_1.MultipartCompressingUploader);
|
|
56
56
|
});
|
|
57
|
-
(0, vitest_1.it)("should
|
|
57
|
+
(0, vitest_1.it)("should compress and upload a single file", async () => {
|
|
58
58
|
await (0, promises_1.writeFile)((0, path_1.join)(testDir, "test.txt"), "Hello World");
|
|
59
59
|
const uploader = createUploader();
|
|
60
60
|
const result = await uploader.execute();
|
|
@@ -62,7 +62,7 @@ vitest_1.vi.mock("@alwaysmeticulous/client", async () => {
|
|
|
62
62
|
(0, vitest_1.expect)(result.eTags.length).toBeGreaterThan(0);
|
|
63
63
|
(0, vitest_1.expect)(uploadedParts.length).toBeGreaterThan(0);
|
|
64
64
|
});
|
|
65
|
-
(0, vitest_1.it)("should
|
|
65
|
+
(0, vitest_1.it)("should compress and upload multiple files", async () => {
|
|
66
66
|
await (0, promises_1.writeFile)((0, path_1.join)(testDir, "file1.txt"), "Content 1");
|
|
67
67
|
await (0, promises_1.writeFile)((0, path_1.join)(testDir, "file2.txt"), "Content 2");
|
|
68
68
|
await (0, promises_1.writeFile)((0, path_1.join)(testDir, "file3.txt"), "Content 3");
|
|
@@ -87,4 +87,4 @@ vitest_1.vi.mock("@alwaysmeticulous/client", async () => {
|
|
|
87
87
|
(0, vitest_1.expect)(result.eTags).toBeDefined();
|
|
88
88
|
});
|
|
89
89
|
});
|
|
90
|
-
//# sourceMappingURL=multipart-
|
|
90
|
+
//# sourceMappingURL=multipart-compressing-uploader.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multipart-compressing-uploader.spec.js","sourceRoot":"","sources":["../../../src/upload-utils/__tests__/multipart-compressing-uploader.spec.ts"],"names":[],"mappings":";;AAAA,0CAAmD;AACnD,2BAA4B;AAC5B,+BAA4B;AAC5B,mCAAyE;AACzE,sFAG2C;AAE3C,WAAE,CAAC,IAAI,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;IAC7C,MAAM,MAAM,GAAG,MAAM,WAAE,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC;IACjE,OAAO;QACL,GAAG,MAAM;QACT,iBAAiB,EAAE,WAAE;aAClB,EAAE,EAAE;aACJ,iBAAiB,CAAC,EAAE,aAAa,EAAE,gCAAgC,EAAE,CAAC;KAC1E,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,8BAA8B,EAAE,GAAG,EAAE;IAC5C,IAAI,OAAe,CAAC;IACpB,MAAM,aAAa,GAAyC,EAAE,CAAC;IAE/D,IAAA,mBAAU,EAAC,KAAK,IAAI,EAAE;QACpB,OAAO,GAAG,IAAA,WAAI,EAAC,IAAA,WAAM,GAAE,EAAE,8BAA8B,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACrE,MAAM,IAAA,gBAAK,EAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,IAAA,kBAAS,EAAC,KAAK,IAAI,EAAE;QACnB,IAAI;YACF,MAAM,IAAA,aAAE,EAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;SACrD;QAAC,MAAM;YACN,wBAAwB;SACzB;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,CACrB,YAAuD,EAAE,EAC3B,EAAE;QAChC,MAAM,IAAI,GAAqC;YAC7C,UAAU,EAAE,OAAO;YACnB,cAAc,EAAE;gBACd,2BAA2B;gBAC3B,2BAA2B;gBAC3B,2BAA2B;aAC5B;YACD,eAAe,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;YACjC,WAAW,EAAE,YAAY;YACzB,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,EAAS;YACjB,uBAAuB,EAAE,KAAK,EAAE,GAAW,EAAE,MAAc,EAAE,EAAE;gBAC7D,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjD,OAAO,QAAQ,aAAa,CAAC,MAAM,EAAE,CAAC;YACxC,CAAC;YACD,GAAG,SAAS;SACb,CAAC;QACF,OAAO,IAAI,6DAA4B,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,IAAA,WAAE,EAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,IAAA,eAAM,EAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,6DAA4B,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,UAAU,CAAC,EAAE,aAAa,CAAC,CAAC;QAE1D,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAExC,IAAA,eAAM,EAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/C,IAAA,eAAM,EAAC,aAAa,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QACzD,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QACzD,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QAEzD,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAExC,IAAA,eAAM,EAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAA,eAAM,EAAC,aAAa,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,IAAA,gBAAK,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,UAAU,CAAC,EAAE,cAAc,CAAC,CAAC;QAC3D,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAEzE,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAExC,IAAA,eAAM,EAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAExC,IAAA,eAAM,EAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/upload-utils/{multipart-zip-uploader.d.ts → multipart-compressing-uploader.d.ts}
RENAMED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import { DeploymentArchiveType } from "@alwaysmeticulous/api";
|
|
4
4
|
import { createClient, MultiPartUploadInfo } from "@alwaysmeticulous/client";
|
|
5
5
|
export declare const UPLOAD_ARCHIVE_FILE_FORMAT: DeploymentArchiveType;
|
|
6
|
-
export interface
|
|
6
|
+
export interface MultipartCompressingUploaderArgs {
|
|
7
7
|
folderPath: string;
|
|
8
8
|
uploadPartUrls: string[];
|
|
9
9
|
uploadChunkSize: number;
|
|
@@ -12,20 +12,15 @@ export interface MultipartZipUploaderArgs {
|
|
|
12
12
|
client: ReturnType<typeof createClient>;
|
|
13
13
|
uploadBufferToSignedUrl: (url: string, buffer: Buffer) => Promise<string>;
|
|
14
14
|
}
|
|
15
|
-
export declare class
|
|
15
|
+
export declare class MultipartCompressingUploader {
|
|
16
16
|
private readonly args;
|
|
17
17
|
private totalUploadedBytes;
|
|
18
18
|
private preSignedUrlIndex;
|
|
19
19
|
private readonly logger;
|
|
20
20
|
private readonly bufferManager;
|
|
21
|
-
constructor(args:
|
|
21
|
+
constructor(args: MultipartCompressingUploaderArgs);
|
|
22
22
|
execute(): Promise<MultiPartUploadInfo>;
|
|
23
|
+
private streamTarCompressed;
|
|
23
24
|
private uploadPart;
|
|
24
25
|
private getUploadUrl;
|
|
25
|
-
private createArchive;
|
|
26
|
-
private createUploadStream;
|
|
27
|
-
private walkAndZipDirectory;
|
|
28
|
-
private getDirectoryEntriesWithStats;
|
|
29
|
-
private processDirectoryEntry;
|
|
30
|
-
private processSymbolicLink;
|
|
31
26
|
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MultipartCompressingUploader = exports.UPLOAD_ARCHIVE_FILE_FORMAT = void 0;
|
|
4
|
+
const zlib_1 = require("zlib");
|
|
5
|
+
const client_1 = require("@alwaysmeticulous/client");
|
|
6
|
+
const common_1 = require("@alwaysmeticulous/common");
|
|
7
|
+
const fast_zlib_1 = require("fast-zlib");
|
|
8
|
+
const tar_1 = require("tar");
|
|
9
|
+
const multipart_buffer_manager_1 = require("./multipart-buffer-manager");
|
|
10
|
+
const MAX_CONCURRENT_UPLOADS = 4;
|
|
11
|
+
exports.UPLOAD_ARCHIVE_FILE_FORMAT = "tar.d";
|
|
12
|
+
/**
|
|
13
|
+
* Compression level of the deflate algorithm. Ranges from 0 (uncompressed)
|
|
14
|
+
* to 9 (maximum compression).
|
|
15
|
+
*/
|
|
16
|
+
const COMPRESSION_LEVEL = 3;
|
|
17
|
+
class MultipartCompressingUploader {
|
|
18
|
+
constructor(args) {
|
|
19
|
+
this.args = args;
|
|
20
|
+
this.totalUploadedBytes = 0;
|
|
21
|
+
this.preSignedUrlIndex = 0;
|
|
22
|
+
this.logger = (0, common_1.initLogger)();
|
|
23
|
+
this.bufferManager = new multipart_buffer_manager_1.MultipartBufferManager({
|
|
24
|
+
uploadChunkSize: args.uploadChunkSize,
|
|
25
|
+
uploadPart: this.uploadPart.bind(this),
|
|
26
|
+
maxConcurrentUploads: MAX_CONCURRENT_UPLOADS,
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
async execute() {
|
|
30
|
+
const deflate = new fast_zlib_1.DeflateRaw({ level: COMPRESSION_LEVEL });
|
|
31
|
+
await this.streamTarCompressed(deflate);
|
|
32
|
+
const finalChunk = deflate.process(Buffer.alloc(0), zlib_1.constants.Z_FINISH);
|
|
33
|
+
if (finalChunk.length > 0) {
|
|
34
|
+
this.bufferManager.addChunk(finalChunk);
|
|
35
|
+
}
|
|
36
|
+
await this.bufferManager.flush(true);
|
|
37
|
+
const eTags = await this.bufferManager.finalize();
|
|
38
|
+
return { awsUploadId: this.args.awsUploadId, eTags };
|
|
39
|
+
}
|
|
40
|
+
streamTarCompressed(deflate) {
|
|
41
|
+
return new Promise((resolve, reject) => {
|
|
42
|
+
const tarStream = (0, tar_1.create)({
|
|
43
|
+
cwd: this.args.folderPath,
|
|
44
|
+
follow: true,
|
|
45
|
+
portable: true,
|
|
46
|
+
}, ["."]);
|
|
47
|
+
tarStream.on("data", (chunk) => {
|
|
48
|
+
const compressed = deflate.process(chunk);
|
|
49
|
+
if (compressed.length > 0) {
|
|
50
|
+
this.bufferManager.addChunk(compressed);
|
|
51
|
+
if (this.bufferManager.getBufferSize() >= this.args.uploadChunkSize) {
|
|
52
|
+
this.bufferManager
|
|
53
|
+
.flush(false)
|
|
54
|
+
.catch(reject);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
tarStream.on("end", resolve);
|
|
59
|
+
tarStream.on("error", reject);
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
async uploadPart(buffer, partNumber, isLastPart) {
|
|
63
|
+
const uploadUrl = await this.getUploadUrl(buffer, partNumber, isLastPart);
|
|
64
|
+
const eTag = await this.args.uploadBufferToSignedUrl(uploadUrl, buffer);
|
|
65
|
+
this.totalUploadedBytes += buffer.length;
|
|
66
|
+
this.logger.info(`Uploaded part ${partNumber} (${buffer.length} bytes, ${this.totalUploadedBytes} total)`);
|
|
67
|
+
return eTag;
|
|
68
|
+
}
|
|
69
|
+
async getUploadUrl(buffer, partNumber, isLastPart) {
|
|
70
|
+
const isFullSizeChunk = buffer.length === this.args.uploadChunkSize;
|
|
71
|
+
const canUsePreSignedUrl = isFullSizeChunk &&
|
|
72
|
+
!isLastPart &&
|
|
73
|
+
this.preSignedUrlIndex < this.args.uploadPartUrls.length;
|
|
74
|
+
if (canUsePreSignedUrl) {
|
|
75
|
+
return this.args.uploadPartUrls[this.preSignedUrlIndex++];
|
|
76
|
+
}
|
|
77
|
+
const { uploadPartUrl } = await (0, client_1.requestUploadPart)({
|
|
78
|
+
client: this.args.client,
|
|
79
|
+
uploadId: this.args.uploadId,
|
|
80
|
+
awsUploadId: this.args.awsUploadId,
|
|
81
|
+
partNumber,
|
|
82
|
+
size: buffer.length,
|
|
83
|
+
archiveType: exports.UPLOAD_ARCHIVE_FILE_FORMAT,
|
|
84
|
+
});
|
|
85
|
+
return uploadPartUrl;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
exports.MultipartCompressingUploader = MultipartCompressingUploader;
|
|
89
|
+
//# sourceMappingURL=multipart-compressing-uploader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multipart-compressing-uploader.js","sourceRoot":"","sources":["../../src/upload-utils/multipart-compressing-uploader.ts"],"names":[],"mappings":";;;AAAA,+BAAkD;AAElD,qDAIkC;AAClC,qDAAsD;AACtD,yCAAuC;AACvC,6BAA0C;AAC1C,yEAAoE;AAEpE,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACpB,QAAA,0BAA0B,GAA0B,OAAO,CAAC;AAEzE;;;GAGG;AACH,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAY5B,MAAa,4BAA4B;IAMvC,YAA6B,IAAsC;QAAtC,SAAI,GAAJ,IAAI,CAAkC;QAL3D,uBAAkB,GAAG,CAAC,CAAC;QACvB,sBAAiB,GAAG,CAAC,CAAC;QACb,WAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;QAIrC,IAAI,CAAC,aAAa,GAAG,IAAI,iDAAsB,CAAC;YAC9C,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;YACtC,oBAAoB,EAAE,sBAAsB;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,GAAG,IAAI,sBAAU,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAE7D,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAExC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAChC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EACf,gBAAa,CAAC,QAAQ,CACvB,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SACzC;QACD,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAErC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;QAClD,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC;IACvD,CAAC;IAEO,mBAAmB,CAAC,OAAmB;QAC7C,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,MAAM,SAAS,GAAG,IAAA,YAAS,EACzB;gBACE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU;gBACzB,MAAM,EAAE,IAAI;gBACZ,QAAQ,EAAE,IAAI;aACf,EACD,CAAC,GAAG,CAAC,CACN,CAAC;YAEF,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBACrC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC1C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;oBACzB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;oBACxC,IAAI,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;wBACnE,IAAI,CAAC,aAAa;6BACf,KAAK,CAAC,KAAK,CAAC;6BACZ,KAAK,CAAC,MAAM,CAAC,CAAC;qBAClB;iBACF;YACH,CAAC,CAAC,CAAC;YAEH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC7B,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,MAAc,EACd,UAAkB,EAClB,UAAmB;QAEnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QAC1E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAExE,IAAI,CAAC,kBAAkB,IAAI,MAAM,CAAC,MAAM,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iBAAiB,UAAU,KAAK,MAAM,CAAC,MAAM,WAAW,IAAI,CAAC,kBAAkB,SAAS,CACzF,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,MAAc,EACd,UAAkB,EAClB,UAAmB;QAEnB,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACpE,MAAM,kBAAkB,GACtB,eAAe;YACf,CAAC,UAAU;YACX,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;QAE3D,IAAI,kBAAkB,EAAE;YACtB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;SAC3D;QAED,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAA,0BAAiB,EAAC;YAChD,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM;YACxB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ;YAC5B,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;YAClC,UAAU;YACV,IAAI,EAAE,MAAM,CAAC,MAAM;YACnB,WAAW,EAAE,kCAA0B;SACxC,CAAC,CAAC;QAEH,OAAO,aAAa,CAAC;IACvB,CAAC;CACF;AAtGD,oEAsGC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@alwaysmeticulous/remote-replay-launcher",
|
|
3
|
-
"version": "2.255.
|
|
3
|
+
"version": "2.255.2",
|
|
4
4
|
"description": "Executes a remote replay on Meticulous' infrastructure",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -21,18 +21,18 @@
|
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
23
|
"@alwaysmeticulous/api": "2.255.0",
|
|
24
|
-
"@alwaysmeticulous/client": "2.255.
|
|
24
|
+
"@alwaysmeticulous/client": "2.255.2",
|
|
25
25
|
"@alwaysmeticulous/common": "2.255.0",
|
|
26
26
|
"@alwaysmeticulous/tunnels-client": "2.255.0",
|
|
27
27
|
"@sentry/node": "^10.3.0",
|
|
28
|
-
"archiver": "^7.0.0",
|
|
29
28
|
"dockerode": "^4.0.2",
|
|
29
|
+
"fast-zlib": "^2.0.1",
|
|
30
30
|
"loglevel": "^1.8.0",
|
|
31
31
|
"p-limit": "^3.1.0",
|
|
32
|
-
"systeminformation": "^5.25.11"
|
|
32
|
+
"systeminformation": "^5.25.11",
|
|
33
|
+
"tar": "^7.0.0"
|
|
33
34
|
},
|
|
34
35
|
"devDependencies": {
|
|
35
|
-
"@types/archiver": "^5.1.1",
|
|
36
36
|
"@types/dockerode": "^4.0.1",
|
|
37
37
|
"vitest": "catalog:"
|
|
38
38
|
},
|
|
@@ -53,5 +53,5 @@
|
|
|
53
53
|
"bugs": {
|
|
54
54
|
"url": "https://github.com/alwaysmeticulous/meticulous-sdk/issues"
|
|
55
55
|
},
|
|
56
|
-
"gitHead": "
|
|
56
|
+
"gitHead": "badbe440d66845b91753c3cd12b1df89b573fd53"
|
|
57
57
|
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"multipart-zip-uploader.spec.js","sourceRoot":"","sources":["../../../src/upload-utils/__tests__/multipart-zip-uploader.spec.ts"],"names":[],"mappings":";;AAAA,0CAAmD;AACnD,2BAA4B;AAC5B,+BAA4B;AAC5B,mCAAyE;AACzE,sEAGmC;AAEnC,WAAE,CAAC,IAAI,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;IAC7C,MAAM,MAAM,GAAG,MAAM,WAAE,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC;IACjE,OAAO;QACL,GAAG,MAAM;QACT,iBAAiB,EAAE,WAAE;aAClB,EAAE,EAAE;aACJ,iBAAiB,CAAC,EAAE,aAAa,EAAE,gCAAgC,EAAE,CAAC;KAC1E,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,IAAI,OAAe,CAAC;IACpB,MAAM,aAAa,GAAyC,EAAE,CAAC;IAE/D,IAAA,mBAAU,EAAC,KAAK,IAAI,EAAE;QACpB,OAAO,GAAG,IAAA,WAAI,EAAC,IAAA,WAAM,GAAE,EAAE,sBAAsB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAA,gBAAK,EAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,IAAA,kBAAS,EAAC,KAAK,IAAI,EAAE;QACnB,IAAI;YACF,MAAM,IAAA,aAAE,EAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;SACrD;QAAC,MAAM;YACN,wBAAwB;SACzB;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,CACrB,YAA+C,EAAE,EAC3B,EAAE;QACxB,MAAM,IAAI,GAA6B;YACrC,UAAU,EAAE,OAAO;YACnB,cAAc,EAAE;gBACd,2BAA2B;gBAC3B,2BAA2B;gBAC3B,2BAA2B;aAC5B;YACD,eAAe,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;YACjC,WAAW,EAAE,YAAY;YACzB,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,EAAS;YACjB,uBAAuB,EAAE,KAAK,EAAE,GAAW,EAAE,MAAc,EAAE,EAAE;gBAC7D,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjD,OAAO,QAAQ,aAAa,CAAC,MAAM,EAAE,CAAC;YACxC,CAAC;YACD,GAAG,SAAS;SACb,CAAC;QACF,OAAO,IAAI,6CAAoB,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC;IAEF,IAAA,WAAE,EAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,IAAA,eAAM,EAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,6CAAoB,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,UAAU,CAAC,EAAE,aAAa,CAAC,CAAC;QAE1D,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAExC,IAAA,eAAM,EAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/C,IAAA,eAAM,EAAC,aAAa,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QACzD,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QACzD,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QAEzD,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAExC,IAAA,eAAM,EAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAA,eAAM,EAAC,aAAa,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,IAAA,gBAAK,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,UAAU,CAAC,EAAE,cAAc,CAAC,CAAC;QAC3D,MAAM,IAAA,oBAAS,EAAC,IAAA,WAAI,EAAC,OAAO,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAEzE,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAExC,IAAA,eAAM,EAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAExC,IAAA,eAAM,EAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.MultipartZipUploader = exports.UPLOAD_ARCHIVE_FILE_FORMAT = void 0;
|
|
7
|
-
const promises_1 = require("fs/promises");
|
|
8
|
-
const path_1 = require("path");
|
|
9
|
-
const stream_1 = require("stream");
|
|
10
|
-
const client_1 = require("@alwaysmeticulous/client");
|
|
11
|
-
const common_1 = require("@alwaysmeticulous/common");
|
|
12
|
-
const archiver_1 = __importDefault(require("archiver"));
|
|
13
|
-
const p_limit_1 = __importDefault(require("p-limit"));
|
|
14
|
-
const multipart_buffer_manager_1 = require("./multipart-buffer-manager");
|
|
15
|
-
const MAX_CONCURRENT_UPLOADS = 4;
|
|
16
|
-
const FS_CONCURRENCY = 10;
|
|
17
|
-
const ZIP_COMPRESSION_LEVEL = 3;
|
|
18
|
-
exports.UPLOAD_ARCHIVE_FILE_FORMAT = "zip";
|
|
19
|
-
const allWithLimit = async (items, limit, handler) => {
|
|
20
|
-
const limited = (0, p_limit_1.default)(limit);
|
|
21
|
-
return Promise.all(items.map((item) => limited(() => handler(item))));
|
|
22
|
-
};
|
|
23
|
-
class MultipartZipUploader {
|
|
24
|
-
constructor(args) {
|
|
25
|
-
this.args = args;
|
|
26
|
-
this.totalUploadedBytes = 0;
|
|
27
|
-
this.preSignedUrlIndex = 0;
|
|
28
|
-
this.logger = (0, common_1.initLogger)();
|
|
29
|
-
this.bufferManager = new multipart_buffer_manager_1.MultipartBufferManager({
|
|
30
|
-
uploadChunkSize: args.uploadChunkSize,
|
|
31
|
-
uploadPart: this.uploadPart.bind(this),
|
|
32
|
-
maxConcurrentUploads: MAX_CONCURRENT_UPLOADS,
|
|
33
|
-
});
|
|
34
|
-
}
|
|
35
|
-
async execute() {
|
|
36
|
-
return new Promise((resolve, reject) => {
|
|
37
|
-
const archive = this.createArchive();
|
|
38
|
-
const uploadStream = this.createUploadStream(resolve, reject);
|
|
39
|
-
archive.on("error", reject);
|
|
40
|
-
uploadStream.on("error", reject);
|
|
41
|
-
archive.pipe(uploadStream);
|
|
42
|
-
this.walkAndZipDirectory(archive).catch(reject);
|
|
43
|
-
});
|
|
44
|
-
}
|
|
45
|
-
async uploadPart(buffer, partNumber, isLastPart) {
|
|
46
|
-
const uploadUrl = await this.getUploadUrl(buffer, partNumber, isLastPart);
|
|
47
|
-
const eTag = await this.args.uploadBufferToSignedUrl(uploadUrl, buffer);
|
|
48
|
-
this.totalUploadedBytes += buffer.length;
|
|
49
|
-
this.logger.info(`Uploaded part ${partNumber} (${buffer.length} bytes, ${this.totalUploadedBytes} total)`);
|
|
50
|
-
return eTag;
|
|
51
|
-
}
|
|
52
|
-
async getUploadUrl(buffer, partNumber, isLastPart) {
|
|
53
|
-
const isFullSizeChunk = buffer.length === this.args.uploadChunkSize;
|
|
54
|
-
const canUsePreSignedUrl = isFullSizeChunk &&
|
|
55
|
-
!isLastPart &&
|
|
56
|
-
this.preSignedUrlIndex < this.args.uploadPartUrls.length;
|
|
57
|
-
if (canUsePreSignedUrl) {
|
|
58
|
-
return this.args.uploadPartUrls[this.preSignedUrlIndex++];
|
|
59
|
-
}
|
|
60
|
-
const { uploadPartUrl } = await (0, client_1.requestUploadPart)({
|
|
61
|
-
client: this.args.client,
|
|
62
|
-
uploadId: this.args.uploadId,
|
|
63
|
-
awsUploadId: this.args.awsUploadId,
|
|
64
|
-
partNumber,
|
|
65
|
-
size: buffer.length,
|
|
66
|
-
});
|
|
67
|
-
return uploadPartUrl;
|
|
68
|
-
}
|
|
69
|
-
createArchive() {
|
|
70
|
-
return (0, archiver_1.default)("zip", {
|
|
71
|
-
zlib: { level: ZIP_COMPRESSION_LEVEL },
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
createUploadStream(resolve, reject) {
|
|
75
|
-
return new stream_1.Writable({
|
|
76
|
-
write: (chunk, _encoding, callback) => {
|
|
77
|
-
this.bufferManager.addChunk(chunk);
|
|
78
|
-
if (this.bufferManager.getBufferSize() >= this.args.uploadChunkSize) {
|
|
79
|
-
this.bufferManager
|
|
80
|
-
.flush(false)
|
|
81
|
-
.then(() => callback())
|
|
82
|
-
.catch(callback);
|
|
83
|
-
}
|
|
84
|
-
else {
|
|
85
|
-
callback();
|
|
86
|
-
}
|
|
87
|
-
},
|
|
88
|
-
final: (callback) => {
|
|
89
|
-
this.bufferManager
|
|
90
|
-
.flush(true)
|
|
91
|
-
.then(() => this.bufferManager.finalize())
|
|
92
|
-
.then((sortedETags) => {
|
|
93
|
-
resolve({ awsUploadId: this.args.awsUploadId, eTags: sortedETags });
|
|
94
|
-
callback();
|
|
95
|
-
})
|
|
96
|
-
.catch((err) => {
|
|
97
|
-
callback(err);
|
|
98
|
-
reject(err);
|
|
99
|
-
});
|
|
100
|
-
},
|
|
101
|
-
});
|
|
102
|
-
}
|
|
103
|
-
async walkAndZipDirectory(archive) {
|
|
104
|
-
const stack = [
|
|
105
|
-
{
|
|
106
|
-
absolutePath: await (0, promises_1.realpath)(this.args.folderPath),
|
|
107
|
-
pathInArchive: "",
|
|
108
|
-
ancestors: new Set(),
|
|
109
|
-
},
|
|
110
|
-
];
|
|
111
|
-
while (stack.length > 0) {
|
|
112
|
-
const { absolutePath, pathInArchive, ancestors } = stack.pop();
|
|
113
|
-
if (ancestors.has(absolutePath)) {
|
|
114
|
-
continue;
|
|
115
|
-
}
|
|
116
|
-
const newAncestors = new Set([...ancestors, absolutePath]);
|
|
117
|
-
const entriesWithStats = await this.getDirectoryEntriesWithStats(absolutePath, pathInArchive);
|
|
118
|
-
for (const entry of entriesWithStats) {
|
|
119
|
-
await this.processDirectoryEntry(archive, entry, newAncestors, stack);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
await archive.finalize();
|
|
123
|
-
}
|
|
124
|
-
async getDirectoryEntriesWithStats(absolutePath, pathInArchive) {
|
|
125
|
-
const entries = await (0, promises_1.readdir)(absolutePath);
|
|
126
|
-
return allWithLimit(entries, FS_CONCURRENCY, async (entry) => {
|
|
127
|
-
const entryAbsolutePath = (0, path_1.join)(absolutePath, entry);
|
|
128
|
-
const entryPathInArchive = (0, path_1.join)(pathInArchive, entry);
|
|
129
|
-
const entryStats = await (0, promises_1.lstat)(entryAbsolutePath);
|
|
130
|
-
return { entryAbsolutePath, entryPathInArchive, entryStats };
|
|
131
|
-
});
|
|
132
|
-
}
|
|
133
|
-
async processDirectoryEntry(archive, entry, ancestors, stack) {
|
|
134
|
-
const { entryAbsolutePath, entryPathInArchive, entryStats } = entry;
|
|
135
|
-
if (entryStats.isSymbolicLink()) {
|
|
136
|
-
await this.processSymbolicLink(archive, entryAbsolutePath, entryPathInArchive, ancestors, stack);
|
|
137
|
-
return;
|
|
138
|
-
}
|
|
139
|
-
if (entryStats.isFile()) {
|
|
140
|
-
archive.file(entryAbsolutePath, { name: entryPathInArchive });
|
|
141
|
-
return;
|
|
142
|
-
}
|
|
143
|
-
if (entryStats.isDirectory()) {
|
|
144
|
-
stack.push({
|
|
145
|
-
absolutePath: entryAbsolutePath,
|
|
146
|
-
pathInArchive: entryPathInArchive,
|
|
147
|
-
ancestors,
|
|
148
|
-
});
|
|
149
|
-
return;
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
async processSymbolicLink(archive, entryAbsolutePath, entryPathInArchive, ancestors, stack) {
|
|
153
|
-
const targetAbsolutePath = await (0, promises_1.realpath)(entryAbsolutePath);
|
|
154
|
-
const targetStats = await (0, promises_1.stat)(targetAbsolutePath);
|
|
155
|
-
if (targetStats.isFile()) {
|
|
156
|
-
archive.file(targetAbsolutePath, { name: entryPathInArchive });
|
|
157
|
-
return;
|
|
158
|
-
}
|
|
159
|
-
if (targetStats.isDirectory() && !ancestors.has(targetAbsolutePath)) {
|
|
160
|
-
stack.push({
|
|
161
|
-
absolutePath: entryAbsolutePath,
|
|
162
|
-
pathInArchive: entryPathInArchive,
|
|
163
|
-
ancestors,
|
|
164
|
-
});
|
|
165
|
-
return;
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
exports.MultipartZipUploader = MultipartZipUploader;
|
|
170
|
-
//# sourceMappingURL=multipart-zip-uploader.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"multipart-zip-uploader.js","sourceRoot":"","sources":["../../src/upload-utils/multipart-zip-uploader.ts"],"names":[],"mappings":";;;;;;AACA,0CAA6D;AAC7D,+BAA4B;AAC5B,mCAAkC;AAElC,qDAIkC;AAClC,qDAAsD;AACtD,wDAAgC;AAChC,sDAA6B;AAC7B,yEAAoE;AAEpE,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAEnB,QAAA,0BAA0B,GAA0B,KAAK,CAAC;AAEvE,MAAM,YAAY,GAAG,KAAK,EACxB,KAAU,EACV,KAAa,EACb,OAAgC,EACT,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,iBAAM,EAAC,KAAK,CAAC,CAAC;IAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC;AAkBF,MAAa,oBAAoB;IAM/B,YAA6B,IAA8B;QAA9B,SAAI,GAAJ,IAAI,CAA0B;QALnD,uBAAkB,GAAG,CAAC,CAAC;QACvB,sBAAiB,GAAG,CAAC,CAAC;QACb,WAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;QAIrC,IAAI,CAAC,aAAa,GAAG,IAAI,iDAAsB,CAAC;YAC9C,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;YACtC,oBAAoB,EAAE,sBAAsB;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO;QACX,OAAO,IAAI,OAAO,CAAsB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAE9D,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC5B,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE3B,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,MAAc,EACd,UAAkB,EAClB,UAAmB;QAEnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QAC1E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAExE,IAAI,CAAC,kBAAkB,IAAI,MAAM,CAAC,MAAM,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iBAAiB,UAAU,KAAK,MAAM,CAAC,MAAM,WAAW,IAAI,CAAC,kBAAkB,SAAS,CACzF,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,MAAc,EACd,UAAkB,EAClB,UAAmB;QAEnB,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACpE,MAAM,kBAAkB,GACtB,eAAe;YACf,CAAC,UAAU;YACX,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;QAE3D,IAAI,kBAAkB,EAAE;YACtB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;SAC3D;QAED,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAA,0BAAiB,EAAC;YAChD,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM;YACxB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ;YAC5B,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;YAClC,UAAU;YACV,IAAI,EAAE,MAAM,CAAC,MAAM;SACpB,CAAC,CAAC;QAEH,OAAO,aAAa,CAAC;IACvB,CAAC;IAEO,aAAa;QACnB,OAAO,IAAA,kBAAQ,EAAC,KAAK,EAAE;YACrB,IAAI,EAAE,EAAE,KAAK,EAAE,qBAAqB,EAAE;SACvC,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB,CACxB,OAA6C,EAC7C,MAAiC;QAEjC,OAAO,IAAI,iBAAQ,CAAC;YAClB,KAAK,EAAE,CAAC,KAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBAC5C,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAEnC,IAAI,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;oBACnE,IAAI,CAAC,aAAa;yBACf,KAAK,CAAC,KAAK,CAAC;yBACZ,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;yBACtB,KAAK,CAAC,QAAQ,CAAC,CAAC;iBACpB;qBAAM;oBACL,QAAQ,EAAE,CAAC;iBACZ;YACH,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAClB,IAAI,CAAC,aAAa;qBACf,KAAK,CAAC,IAAI,CAAC;qBACX,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;qBACzC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;oBACpB,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;oBACpE,QAAQ,EAAE,CAAC;gBACb,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACb,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACd,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,OAA0B;QAC1D,MAAM,KAAK,GAA0B;YACnC;gBACE,YAAY,EAAE,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;gBAClD,aAAa,EAAE,EAAE;gBACjB,SAAS,EAAE,IAAI,GAAG,EAAU;aAC7B;SACF,CAAC;QAEF,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YACvB,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;YAChE,IAAI,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;gBAC/B,SAAS;aACV;YAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;YAC3D,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAC9D,YAAY,EACZ,aAAa,CACd,CAAC;YAEF,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE;gBACpC,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;aACvE;SACF;QAED,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,4BAA4B,CACxC,YAAoB,EACpB,aAAqB;QAErB,MAAM,OAAO,GAAG,MAAM,IAAA,kBAAO,EAAC,YAAY,CAAC,CAAC;QAC5C,OAAO,YAAY,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC3D,MAAM,iBAAiB,GAAG,IAAA,WAAI,EAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YACpD,MAAM,kBAAkB,GAAG,IAAA,WAAI,EAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YACtD,MAAM,UAAU,GAAG,MAAM,IAAA,gBAAK,EAAC,iBAAiB,CAAC,CAAC;YAClD,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,UAAU,EAAE,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,qBAAqB,CACjC,OAA0B,EAC1B,KAIC,EACD,SAAsB,EACtB,KAA4B;QAE5B,MAAM,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC;QAEpE,IAAI,UAAU,CAAC,cAAc,EAAE,EAAE;YAC/B,MAAM,IAAI,CAAC,mBAAmB,CAC5B,OAAO,EACP,iBAAiB,EACjB,kBAAkB,EAClB,SAAS,EACT,KAAK,CACN,CAAC;YACF,OAAO;SACR;QAED,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE;YACvB,OAAO,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAC9D,OAAO;SACR;QAED,IAAI,UAAU,CAAC,WAAW,EAAE,EAAE;YAC5B,KAAK,CAAC,IAAI,CAAC;gBACT,YAAY,EAAE,iBAAiB;gBAC/B,aAAa,EAAE,kBAAkB;gBACjC,SAAS;aACV,CAAC,CAAC;YACH,OAAO;SACR;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,OAA0B,EAC1B,iBAAyB,EACzB,kBAA0B,EAC1B,SAAsB,EACtB,KAA4B;QAE5B,MAAM,kBAAkB,GAAG,MAAM,IAAA,mBAAQ,EAAC,iBAAiB,CAAC,CAAC;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAA,eAAI,EAAC,kBAAkB,CAAC,CAAC;QAEnD,IAAI,WAAW,CAAC,MAAM,EAAE,EAAE;YACxB,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAC/D,OAAO;SACR;QAED,IAAI,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE;YACnE,KAAK,CAAC,IAAI,CAAC;gBACT,YAAY,EAAE,iBAAiB;gBAC/B,aAAa,EAAE,kBAAkB;gBACjC,SAAS;aACV,CAAC,CAAC;YACH,OAAO;SACR;IACH,CAAC;CACF;AApND,oDAoNC"}
|
|
File without changes
|