@automattic/vip 4.1.6 → 4.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/dev-env.lando.template.yml.ejs +1 -1
- package/dist/bin/vip-app-deploy-token-generate.js +110 -78
- package/dist/bin/vip-app-deploy-token.js +24 -16
- package/dist/bin/vip-app-deploy-validate.js +45 -41
- package/dist/bin/vip-app-deploy.js +222 -215
- package/dist/bin/vip-app-list.js +59 -43
- package/dist/bin/vip-app.js +97 -66
- package/dist/bin/vip-backup-db.js +36 -29
- package/dist/bin/vip-backup.js +16 -9
- package/dist/bin/vip-cache-purge-url.js +93 -55
- package/dist/bin/vip-cache.js +23 -14
- package/dist/bin/vip-config-envvar-delete.js +74 -73
- package/dist/bin/vip-config-envvar-get-all.js +68 -68
- package/dist/bin/vip-config-envvar-get.js +46 -46
- package/dist/bin/vip-config-envvar-list.js +54 -57
- package/dist/bin/vip-config-envvar-set.js +102 -96
- package/dist/bin/vip-config-envvar.js +38 -24
- package/dist/bin/vip-config-software-get.js +88 -75
- package/dist/bin/vip-config-software-update.js +116 -120
- package/dist/bin/vip-config-software.js +31 -21
- package/dist/bin/vip-config.js +25 -17
- package/dist/bin/vip-db-phpmyadmin.js +40 -39
- package/dist/bin/vip-db.js +13 -12
- package/dist/bin/vip-defensive-mode-configure.js +197 -197
- package/dist/bin/vip-defensive-mode-disable.js +43 -35
- package/dist/bin/vip-defensive-mode-enable.js +47 -38
- package/dist/bin/vip-defensive-mode.js +28 -17
- package/dist/bin/vip-dev-env-create.js +147 -104
- package/dist/bin/vip-dev-env-destroy.js +48 -39
- package/dist/bin/vip-dev-env-envvar-delete.js +60 -49
- package/dist/bin/vip-dev-env-envvar-get-all.js +52 -45
- package/dist/bin/vip-dev-env-envvar-get.js +48 -37
- package/dist/bin/vip-dev-env-envvar-list.js +48 -41
- package/dist/bin/vip-dev-env-envvar-set.js +83 -70
- package/dist/bin/vip-dev-env-envvar.js +38 -23
- package/dist/bin/vip-dev-env-exec.js +81 -74
- package/dist/bin/vip-dev-env-import-media.js +34 -26
- package/dist/bin/vip-dev-env-import-sql.js +65 -51
- package/dist/bin/vip-dev-env-import.js +23 -14
- package/dist/bin/vip-dev-env-info.js +59 -50
- package/dist/bin/vip-dev-env-list.js +34 -31
- package/dist/bin/vip-dev-env-logs.js +59 -49
- package/dist/bin/vip-dev-env-purge.js +71 -63
- package/dist/bin/vip-dev-env-shell.js +82 -71
- package/dist/bin/vip-dev-env-start.js +77 -65
- package/dist/bin/vip-dev-env-stop.js +80 -74
- package/dist/bin/vip-dev-env-sync-sql.js +80 -72
- package/dist/bin/vip-dev-env-sync.js +18 -11
- package/dist/bin/vip-dev-env-update.js +93 -84
- package/dist/bin/vip-dev-env.js +25 -8
- package/dist/bin/vip-edge-workers-build.js +82 -54
- package/dist/bin/vip-edge-workers-delete.js +84 -52
- package/dist/bin/vip-edge-workers-deploy.js +186 -126
- package/dist/bin/vip-edge-workers-disable.js +75 -44
- package/dist/bin/vip-edge-workers-enable.js +86 -54
- package/dist/bin/vip-edge-workers-get.js +108 -88
- package/dist/bin/vip-edge-workers-init.js +83 -53
- package/dist/bin/vip-edge-workers-list.js +89 -54
- package/dist/bin/vip-edge-workers-new.js +98 -69
- package/dist/bin/vip-edge-workers-validate.js +123 -82
- package/dist/bin/vip-edge-workers.js +19 -6
- package/dist/bin/vip-export-sql.js +75 -69
- package/dist/bin/vip-export.js +12 -7
- package/dist/bin/vip-import-media-abort.js +99 -73
- package/dist/bin/vip-import-media-status.js +83 -51
- package/dist/bin/vip-import-media.js +165 -166
- package/dist/bin/vip-import-sql-status.js +75 -45
- package/dist/bin/vip-import-sql.js +536 -596
- package/dist/bin/vip-import-validate-files.js +122 -141
- package/dist/bin/vip-import-validate-sql.js +52 -16
- package/dist/bin/vip-import.js +16 -7
- package/dist/bin/vip-logout.js +17 -13
- package/dist/bin/vip-logs.js +209 -183
- package/dist/bin/vip-search-replace.js +85 -53
- package/dist/bin/vip-slowlogs.js +168 -147
- package/dist/bin/vip-sync.js +163 -133
- package/dist/bin/vip-whoami.js +71 -35
- package/dist/bin/vip-wp.js +506 -549
- package/dist/bin/vip.js +161 -116
- package/dist/commands/backup-db.js +183 -149
- package/dist/commands/dev-env-import-sql.js +162 -117
- package/dist/commands/dev-env-sync-sql.js +372 -330
- package/dist/commands/export-sql.js +441 -425
- package/dist/commands/phpmyadmin.js +232 -212
- package/dist/commands/wp-ssh.js +218 -187
- package/dist/lib/analytics/clients/tracks.js +84 -91
- package/dist/lib/analytics/index.js +27 -27
- package/dist/lib/api/app.js +37 -36
- package/dist/lib/api/cache-purge.js +17 -20
- package/dist/lib/api/constants.js +5 -6
- package/dist/lib/api/edge-workers.js +84 -135
- package/dist/lib/api/error-debug.js +10 -13
- package/dist/lib/api/feature-flags.js +16 -17
- package/dist/lib/api/http.js +47 -48
- package/dist/lib/api/user.js +17 -18
- package/dist/lib/api.js +107 -116
- package/dist/lib/app-logs/app-logs.js +24 -25
- package/dist/lib/app-slowlogs/app-slowlogs.js +23 -24
- package/dist/lib/app-slowlogs/types.js +2 -1
- package/dist/lib/app.js +6 -8
- package/dist/lib/backup-storage-availability/backup-storage-availability.js +119 -119
- package/dist/lib/backup-storage-availability/docker-machine-not-found-error.js +5 -6
- package/dist/lib/cli/apiConfig.js +71 -41
- package/dist/lib/cli/command.js +1120 -873
- package/dist/lib/cli/config.js +29 -29
- package/dist/lib/cli/envAlias.js +35 -50
- package/dist/lib/cli/exit.js +19 -19
- package/dist/lib/cli/format.js +172 -172
- package/dist/lib/cli/progress.js +193 -236
- package/dist/lib/cli/prompt.js +14 -15
- package/dist/lib/cli/runtime-mode.js +5 -6
- package/dist/lib/client-file-uploader.js +489 -641
- package/dist/lib/config/software.js +206 -212
- package/dist/lib/constants/dev-environment.js +40 -37
- package/dist/lib/constants/file-size.js +6 -7
- package/dist/lib/constants/vipgo.js +24 -12
- package/dist/lib/custom-deploy/custom-deploy.js +138 -120
- package/dist/lib/database.js +88 -85
- package/dist/lib/defensive-mode/api.js +41 -46
- package/dist/lib/defensive-mode/cli-helpers.js +40 -40
- package/dist/lib/dev-environment/dev-environment-cli.js +776 -735
- package/dist/lib/dev-environment/dev-environment-configuration-file.js +224 -191
- package/dist/lib/dev-environment/dev-environment-core.js +697 -732
- package/dist/lib/dev-environment/dev-environment-database.js +57 -41
- package/dist/lib/dev-environment/dev-environment-lando.js +734 -666
- package/dist/lib/dev-environment/docker-utils.js +96 -90
- package/dist/lib/dev-environment/env-vars.js +61 -59
- package/dist/lib/dev-environment/lando-loader.js +6 -7
- package/dist/lib/dev-environment/types.js +2 -1
- package/dist/lib/edge-workers/confirmation.js +42 -36
- package/dist/lib/edge-workers/deployment.js +150 -159
- package/dist/lib/edge-workers/index.js +68 -85
- package/dist/lib/edge-workers/location.js +26 -26
- package/dist/lib/edge-workers/output.js +14 -15
- package/dist/lib/edge-workers/project.js +123 -132
- package/dist/lib/edge-workers/toolchains/assemblyscript/constants.js +6 -8
- package/dist/lib/edge-workers/toolchains/assemblyscript/index.js +100 -101
- package/dist/lib/edge-workers/toolchains/assemblyscript/templates.js +31 -33
- package/dist/lib/edge-workers/toolchains/index.js +15 -15
- package/dist/lib/edge-workers/types.js +10 -38
- package/dist/lib/edge-workers/validation.js +161 -150
- package/dist/lib/env.js +16 -17
- package/dist/lib/envvar/api-delete.js +16 -18
- package/dist/lib/envvar/api-get-all.js +15 -19
- package/dist/lib/envvar/api-get.js +8 -9
- package/dist/lib/envvar/api-list.js +16 -21
- package/dist/lib/envvar/api-set.js +20 -19
- package/dist/lib/envvar/api.js +34 -29
- package/dist/lib/envvar/input.js +29 -34
- package/dist/lib/envvar/logging.js +9 -11
- package/dist/lib/envvar/read-file.js +6 -7
- package/dist/lib/http/download-file.js +76 -44
- package/dist/lib/http/proxy-agent.js +37 -40
- package/dist/lib/http/proxy-dispatcher.js +69 -71
- package/dist/lib/keychain/insecure.js +40 -36
- package/dist/lib/keychain/keychain.js +2 -1
- package/dist/lib/keychain/secure.js +16 -15
- package/dist/lib/keychain.js +22 -22
- package/dist/lib/live-backup-copy.js +117 -115
- package/dist/lib/logout.js +17 -19
- package/dist/lib/media-import/config.js +15 -14
- package/dist/lib/media-import/media-file-import.js +9 -13
- package/dist/lib/media-import/progress.js +53 -65
- package/dist/lib/media-import/status.js +241 -256
- package/dist/lib/media-import/utils.js +15 -15
- package/dist/lib/node-version-check.js +16 -16
- package/dist/lib/promise.js +16 -17
- package/dist/lib/read-file.js +48 -16
- package/dist/lib/rechallenge/client.js +37 -43
- package/dist/lib/rechallenge/errors.js +38 -37
- package/dist/lib/rechallenge/flow.js +148 -123
- package/dist/lib/rechallenge/index.js +23 -24
- package/dist/lib/rechallenge/link.js +153 -151
- package/dist/lib/rechallenge/open-browser.js +14 -17
- package/dist/lib/rechallenge/token-cache.js +83 -86
- package/dist/lib/rechallenge/types.js +5 -14
- package/dist/lib/retry.js +93 -129
- package/dist/lib/search-and-replace.js +164 -148
- package/dist/lib/site-import/db-file-import.js +12 -16
- package/dist/lib/site-import/status.js +291 -291
- package/dist/lib/token.js +86 -87
- package/dist/lib/tracker.js +51 -62
- package/dist/lib/types/graphql/rate-limit-exceeded-error.js +2 -1
- package/dist/lib/types.js +2 -1
- package/dist/lib/user-error.js +6 -8
- package/dist/lib/utils.js +81 -87
- package/dist/lib/validations/custom-deploy.js +218 -186
- package/dist/lib/validations/is-multi-site-sql-dump.js +6 -6
- package/dist/lib/validations/is-multi-site.js +83 -50
- package/dist/lib/validations/is-multisite-domain-mapped.js +92 -62
- package/dist/lib/validations/line-by-line.js +78 -46
- package/dist/lib/validations/site-type.js +59 -63
- package/dist/lib/validations/sql-insert-parser.js +415 -0
- package/dist/lib/validations/sql.js +551 -431
- package/dist/lib/validations/utils.js +27 -29
- package/dist/lib/vip-import-validate-files.js +481 -490
- package/dist/lib/wp/helpers.js +41 -45
- package/dist/lib/wp/readline.js +21 -57
- package/dist/lib/xdg-data.js +11 -12
- package/helpers/mark-cli-executable.js +9 -0
- package/node_modules/@apollo/client/CHANGELOG.md +10 -0
- package/node_modules/@apollo/client/__cjs/cache/inmemory/helpers.cjs +5 -0
- package/node_modules/@apollo/client/__cjs/cache/inmemory/helpers.cjs.map +1 -1
- package/node_modules/@apollo/client/__cjs/cache/inmemory/inMemoryCache.cjs +2 -1
- package/node_modules/@apollo/client/__cjs/cache/inmemory/inMemoryCache.cjs.map +1 -1
- package/node_modules/@apollo/client/__cjs/cache/inmemory/key-extractor.cjs +14 -2
- package/node_modules/@apollo/client/__cjs/cache/inmemory/key-extractor.cjs.map +1 -1
- package/node_modules/@apollo/client/__cjs/cache/inmemory/policies.cjs +16 -0
- package/node_modules/@apollo/client/__cjs/cache/inmemory/policies.cjs.map +1 -1
- package/node_modules/@apollo/client/__cjs/cache/inmemory/policies.d.cts +2 -0
- package/node_modules/@apollo/client/__cjs/cache/inmemory/writeToStore.cjs +3 -2
- package/node_modules/@apollo/client/__cjs/cache/inmemory/writeToStore.cjs.map +1 -1
- package/node_modules/@apollo/client/__cjs/core/QueryInfo.cjs +9 -1
- package/node_modules/@apollo/client/__cjs/core/QueryInfo.cjs.map +1 -1
- package/node_modules/@apollo/client/__cjs/core/QueryInfo.d.cts +1 -1
- package/node_modules/@apollo/client/__cjs/core/QueryManager.cjs +10 -3
- package/node_modules/@apollo/client/__cjs/core/QueryManager.cjs.map +1 -1
- package/node_modules/@apollo/client/__cjs/react/hooks/useFragment.cjs.map +1 -1
- package/node_modules/@apollo/client/__cjs/react/hooks/useFragment.d.cts +42 -43
- package/node_modules/@apollo/client/__cjs/version.cjs +1 -1
- package/node_modules/@apollo/client/cache/inmemory/helpers.js +5 -0
- package/node_modules/@apollo/client/cache/inmemory/helpers.js.map +1 -1
- package/node_modules/@apollo/client/cache/inmemory/inMemoryCache.js +2 -1
- package/node_modules/@apollo/client/cache/inmemory/inMemoryCache.js.map +1 -1
- package/node_modules/@apollo/client/cache/inmemory/key-extractor.js +14 -2
- package/node_modules/@apollo/client/cache/inmemory/key-extractor.js.map +1 -1
- package/node_modules/@apollo/client/cache/inmemory/policies.d.ts +2 -0
- package/node_modules/@apollo/client/cache/inmemory/policies.js +17 -1
- package/node_modules/@apollo/client/cache/inmemory/policies.js.map +1 -1
- package/node_modules/@apollo/client/cache/inmemory/writeToStore.js +3 -2
- package/node_modules/@apollo/client/cache/inmemory/writeToStore.js.map +1 -1
- package/node_modules/@apollo/client/core/QueryInfo.d.ts +1 -1
- package/node_modules/@apollo/client/core/QueryInfo.js +10 -2
- package/node_modules/@apollo/client/core/QueryInfo.js.map +1 -1
- package/node_modules/@apollo/client/core/QueryManager.js +10 -3
- package/node_modules/@apollo/client/core/QueryManager.js.map +1 -1
- package/node_modules/@apollo/client/package.json +1 -1
- package/node_modules/@apollo/client/react/hooks/useFragment.d.ts +42 -43
- package/node_modules/@apollo/client/react/hooks/useFragment.js.map +1 -1
- package/node_modules/@apollo/client/react/hooks-compiled/useFragment.d.ts +42 -43
- package/node_modules/@apollo/client/react/hooks-compiled/useFragment.js.map +1 -1
- package/node_modules/@apollo/client/version.js +1 -1
- package/node_modules/@automattic/vip-search-replace/README.md +6 -1
- package/node_modules/@automattic/vip-search-replace/lib/index.js +46 -14
- package/node_modules/@automattic/vip-search-replace/package.json +1 -1
- package/node_modules/@types/node/README.md +1 -1
- package/node_modules/@types/node/package.json +2 -2
- package/node_modules/@types/node/zlib.d.ts +752 -0
- package/node_modules/check-disk-space/README.md +92 -32
- package/node_modules/check-disk-space/dist/check-disk-space.d.mts +58 -0
- package/node_modules/check-disk-space/dist/check-disk-space.mjs +170 -190
- package/node_modules/check-disk-space/package.json +43 -71
- package/node_modules/cpu-features/build/Makefile +2 -2
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/filesystem.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_aarch64_linux_or_android.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_aarch64_macos_or_iphone.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_aarch64_windows.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_arm_linux_or_android.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_mips_linux_or_android.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_ppc_linux.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_x86_freebsd.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_x86_linux_or_android.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_x86_macos.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/impl_x86_windows.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/stack_line_reader.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpu_features/deps/cpu_features/src/string_view.o.d +1 -1
- package/node_modules/cpu-features/build/Release/.deps/Release/obj.target/cpufeatures/src/binding.o.d +157 -145
- package/node_modules/cpu-features/build/Release/cpufeatures.node +0 -0
- package/node_modules/cpu-features/build/Release/obj.target/cpufeatures/src/binding.o +0 -0
- package/node_modules/cpu-features/build/Release/obj.target/cpufeatures.node +0 -0
- package/node_modules/cpu-features/build/config.gypi +100 -31
- package/node_modules/cpu-features/build/cpufeatures.target.mk +14 -14
- package/node_modules/cpu-features/build/deps/cpu_features/cpu_features.target.mk +14 -14
- package/node_modules/ssh2/lib/protocol/crypto/build/Makefile +2 -2
- package/node_modules/ssh2/lib/protocol/crypto/build/Release/.deps/Release/obj.target/sshcrypto/src/binding.o.d.raw +136 -0
- package/node_modules/ssh2/lib/protocol/crypto/build/sshcrypto.target.mk +14 -14
- package/node_modules/undici/docs/docs/api/Interceptors.md +15 -2
- package/node_modules/undici/lib/dispatcher/client-h2.js +78 -3
- package/node_modules/undici/lib/dispatcher/pool-base.js +20 -3
- package/node_modules/undici/package.json +3 -3
- package/node_modules/undici/types/api.d.ts +1 -1
- package/node_modules/undici/types/utility.d.ts +4 -4
- package/package.json +44 -13
- package/dist/lib/analytics/clients/client.js +0 -1
- package/dist/lib/analytics/clients/pendo.js +0 -78
- package/helpers/check-version.js +0 -16
- package/node_modules/check-disk-space/dist/check-disk-space.cjs +0 -210
- package/node_modules/check-disk-space/dist/check-disk-space.d.ts +0 -53
- package/node_modules/fetch-retry/.eslintignore +0 -1
- package/node_modules/fetch-retry/.eslintrc.yml +0 -20
- package/node_modules/fetch-retry/.github/workflows/node.js.yml +0 -32
- package/node_modules/fetch-retry/.husky/pre-commit +0 -6
- package/node_modules/fetch-retry/.nycrc.json +0 -24
- package/node_modules/fetch-retry/.travis.yml +0 -3
- package/node_modules/fetch-retry/LICENSE +0 -21
- package/node_modules/fetch-retry/README.md +0 -128
- package/node_modules/fetch-retry/dist/fetch-retry.umd.js +0 -151
- package/node_modules/fetch-retry/index.d.ts +0 -29
- package/node_modules/fetch-retry/index.js +0 -143
- package/node_modules/fetch-retry/package.json +0 -41
- package/node_modules/fetch-retry/rollup.config.js +0 -12
- package/node_modules/fetch-retry/test/.eslintrc.yml +0 -7
- package/node_modules/fetch-retry/test/integration/index.js +0 -212
- package/node_modules/fetch-retry/test/integration/mock-api/index.js +0 -42
- package/node_modules/fetch-retry/test/unit/index.js +0 -1169
- package/node_modules/ssh2/lib/protocol/crypto/build/Release/.deps/Release/obj.target/sshcrypto/src/binding.o.d +0 -265
- package/node_modules/ssh2/lib/protocol/crypto/build/Release/.deps/Release/obj.target/sshcrypto.node.d +0 -1
- package/node_modules/ssh2/lib/protocol/crypto/build/Release/.deps/Release/sshcrypto.node.d +0 -1
- package/node_modules/ssh2/lib/protocol/crypto/build/Release/obj.target/sshcrypto/src/binding.o +0 -0
- package/node_modules/ssh2/lib/protocol/crypto/build/Release/obj.target/sshcrypto.node +0 -0
- package/node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node +0 -0
|
@@ -1,87 +1,82 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
exports
|
|
6
|
-
exports.
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.unzipFile = exports.getFileHash = exports.MULTIPART_THRESHOLD = exports.COMPRESS_THRESHOLD = void 0;
|
|
7
|
+
exports.parseEtagHeader = parseEtagHeader;
|
|
7
8
|
exports.fetchWithRetry = fetchWithRetry;
|
|
8
|
-
exports.getFileHash = void 0;
|
|
9
9
|
exports.getFileMeta = getFileMeta;
|
|
10
|
-
exports.
|
|
11
|
-
exports.getPartBoundaries = getPartBoundaries;
|
|
10
|
+
exports.uploadImportFileToS3 = uploadImportFileToS3;
|
|
12
11
|
exports.getSignedUploadRequestData = getSignedUploadRequestData;
|
|
12
|
+
exports.checkFileAccess = checkFileAccess;
|
|
13
13
|
exports.isFile = isFile;
|
|
14
|
-
exports.
|
|
15
|
-
exports.
|
|
16
|
-
exports.
|
|
14
|
+
exports.getFileSize = getFileSize;
|
|
15
|
+
exports.detectCompressedMimeType = detectCompressedMimeType;
|
|
16
|
+
exports.getPartBoundaries = getPartBoundaries;
|
|
17
17
|
exports.uploadParts = uploadParts;
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
18
|
+
const chalk_1 = __importDefault(require("chalk"));
|
|
19
|
+
const crypto_1 = require("crypto");
|
|
20
|
+
const debug_1 = __importDefault(require("debug"));
|
|
21
|
+
const fs_1 = require("fs");
|
|
22
|
+
const promises_1 = require("node:fs/promises");
|
|
23
|
+
const promises_2 = require("node:stream/promises");
|
|
24
|
+
const node_timers_1 = require("node:timers");
|
|
25
|
+
const promises_3 = require("node:timers/promises");
|
|
26
|
+
const os_1 = __importDefault(require("os"));
|
|
27
|
+
const path_1 = __importDefault(require("path"));
|
|
28
|
+
const stream_1 = require("stream");
|
|
29
|
+
const undici_1 = require("undici");
|
|
30
|
+
const xml2js_1 = require("xml2js");
|
|
31
|
+
const zlib_1 = require("zlib");
|
|
32
|
+
const http_1 = __importDefault(require("../lib/api/http"));
|
|
33
|
+
const file_size_1 = require("../lib/constants/file-size");
|
|
35
34
|
function parseEtagHeader(etag) {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
35
|
+
const normalizedEtag = etag.replace(/^W\//u, '').trim();
|
|
36
|
+
if (normalizedEtag.startsWith('"') && normalizedEtag.endsWith('"')) {
|
|
37
|
+
try {
|
|
38
|
+
return JSON.parse(normalizedEtag);
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
debug(`Unable to JSON.parse ETag header, falling back to raw value: ${err.message}`);
|
|
42
|
+
}
|
|
42
43
|
}
|
|
43
|
-
|
|
44
|
-
return normalizedEtag.replace(/^"(.*)"$/u, '$1');
|
|
44
|
+
return normalizedEtag.replace(/^"(.*)"$/u, '$1');
|
|
45
45
|
}
|
|
46
|
-
const isStreamBody = body => typeof body?.pipe === 'function';
|
|
46
|
+
const isStreamBody = (body) => typeof body?.pipe === 'function';
|
|
47
47
|
function createProgressTransform(onChunk) {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
48
|
+
return new stream_1.Transform({
|
|
49
|
+
transform(chunk, _encoding, callback) {
|
|
50
|
+
onChunk(chunk);
|
|
51
|
+
callback(null, chunk);
|
|
52
|
+
},
|
|
53
|
+
});
|
|
54
54
|
}
|
|
55
55
|
function omitUnsupportedFetchHeaders(init) {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
...init,
|
|
63
|
-
headers
|
|
64
|
-
};
|
|
56
|
+
if (!init.headers) {
|
|
57
|
+
return init;
|
|
58
|
+
}
|
|
59
|
+
const headers = new undici_1.Headers(init.headers);
|
|
60
|
+
headers.delete('expect');
|
|
61
|
+
return { ...init, headers };
|
|
65
62
|
}
|
|
66
63
|
function withCauseMessage(err) {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
cause: err
|
|
82
|
-
});
|
|
64
|
+
if (!(err instanceof Error)) {
|
|
65
|
+
return err;
|
|
66
|
+
}
|
|
67
|
+
const cause = err.cause;
|
|
68
|
+
let causeMessage;
|
|
69
|
+
if (cause instanceof Error) {
|
|
70
|
+
causeMessage = cause.message;
|
|
71
|
+
}
|
|
72
|
+
else if (typeof cause === 'string') {
|
|
73
|
+
causeMessage = cause;
|
|
74
|
+
}
|
|
75
|
+
if (!causeMessage || err.message.includes(causeMessage)) {
|
|
76
|
+
return err;
|
|
77
|
+
}
|
|
78
|
+
return new Error(`${err.message}: ${causeMessage}`, { cause: err });
|
|
83
79
|
}
|
|
84
|
-
|
|
85
80
|
/**
|
|
86
81
|
* Wraps `fetch` with exponential-backoff retries.
|
|
87
82
|
*
|
|
@@ -97,77 +92,67 @@ function withCauseMessage(err) {
|
|
|
97
92
|
* once so the underlying error surfaces instead of the misleading undici one.
|
|
98
93
|
*/
|
|
99
94
|
async function fetchWithRetry(input, init = {}, retries = 3, createBody) {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
95
|
+
const bodyIsStream = isStreamBody(init.body);
|
|
96
|
+
// Only retry when we can hand `fetch` a fresh, replayable body each attempt.
|
|
97
|
+
const maxAttempts = createBody || !bodyIsStream ? retries : 0;
|
|
98
|
+
for (let attempt = 0; attempt <= maxAttempts; attempt++) {
|
|
99
|
+
const requestInit = createBody
|
|
100
|
+
? { ...init, body: createBody() }
|
|
101
|
+
: { ...init };
|
|
102
|
+
if (isStreamBody(requestInit.body) && !requestInit.duplex) {
|
|
103
|
+
requestInit.duplex = 'half';
|
|
104
|
+
}
|
|
105
|
+
try {
|
|
106
|
+
// eslint-disable-next-line no-await-in-loop
|
|
107
|
+
return await (0, undici_1.fetch)(input, omitUnsupportedFetchHeaders(requestInit));
|
|
108
|
+
}
|
|
109
|
+
catch (err) {
|
|
110
|
+
if (attempt === maxAttempts) {
|
|
111
|
+
throw withCauseMessage(err);
|
|
112
|
+
}
|
|
113
|
+
// eslint-disable-next-line no-await-in-loop
|
|
114
|
+
await (0, promises_3.setTimeout)(Math.pow(2, attempt) * 1000); // 1000, 2000, 4000
|
|
115
|
+
}
|
|
112
116
|
}
|
|
113
|
-
|
|
114
|
-
// eslint-disable-next-line no-await-in-loop
|
|
115
|
-
return await (0, _undici.fetch)(input, omitUnsupportedFetchHeaders(requestInit));
|
|
116
|
-
} catch (err) {
|
|
117
|
-
if (attempt === maxAttempts) {
|
|
118
|
-
throw withCauseMessage(err);
|
|
119
|
-
}
|
|
120
|
-
// eslint-disable-next-line no-await-in-loop
|
|
121
|
-
await (0, _promises3.setTimeout)(Math.pow(2, attempt) * 1000); // 1000, 2000, 4000
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
throw new Error('unreachable');
|
|
117
|
+
throw new Error('unreachable');
|
|
125
118
|
}
|
|
126
|
-
const debug = (0,
|
|
127
|
-
|
|
119
|
+
const debug = (0, debug_1.default)('vip:lib/client-file-uploader');
|
|
128
120
|
// Files smaller than COMPRESS_THRESHOLD will not be compressed before upload
|
|
129
|
-
|
|
130
|
-
|
|
121
|
+
exports.COMPRESS_THRESHOLD = 16 * file_size_1.MB_IN_BYTES;
|
|
131
122
|
// Files smaller than MULTIPART_THRESHOLD will use `PutObject` vs Multipart Uploads
|
|
132
|
-
|
|
133
|
-
|
|
123
|
+
exports.MULTIPART_THRESHOLD = 32 * file_size_1.MB_IN_BYTES;
|
|
134
124
|
// This is how big each part of a Multipart Upload is (except the last / remainder)
|
|
135
|
-
const UPLOAD_PART_SIZE = 16 *
|
|
136
|
-
|
|
125
|
+
const UPLOAD_PART_SIZE = 16 * file_size_1.MB_IN_BYTES;
|
|
137
126
|
// How many parts will upload at the same time
|
|
138
127
|
const MAX_CONCURRENT_PART_UPLOADS = 5;
|
|
139
|
-
|
|
140
128
|
// How many times a presigned URL request is retried after a network error, 429 or 5xx
|
|
141
129
|
const SIGNED_REQUEST_RETRIES = 3;
|
|
142
|
-
|
|
143
|
-
// TODO: Replace with a proper definitions once we convert lib/cli/command.js to TypeScript
|
|
144
|
-
|
|
145
|
-
const getWorkingTempDir = () => (0, _promises.mkdtemp)(_path.default.join(_os.default.tmpdir(), 'vip-client-file-uploader'));
|
|
130
|
+
const getWorkingTempDir = () => (0, promises_1.mkdtemp)(path_1.default.join(os_1.default.tmpdir(), 'vip-client-file-uploader'));
|
|
146
131
|
const getFileHash = async (fileName, hashType = 'md5') => {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
132
|
+
const src = (0, fs_1.createReadStream)(fileName);
|
|
133
|
+
const dst = (0, crypto_1.createHash)(hashType);
|
|
134
|
+
try {
|
|
135
|
+
await (0, promises_2.pipeline)(src, dst);
|
|
136
|
+
return dst.digest().toString('hex');
|
|
137
|
+
}
|
|
138
|
+
catch (err) {
|
|
139
|
+
throw new Error(`Could not generate file hash: ${err.message}`, {
|
|
140
|
+
cause: err,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
finally {
|
|
144
|
+
src.close();
|
|
145
|
+
}
|
|
159
146
|
};
|
|
160
147
|
exports.getFileHash = getFileHash;
|
|
161
148
|
const gzipFile = async (uncompressedFileName, compressedFileName) => {
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
}
|
|
168
|
-
}
|
|
149
|
+
try {
|
|
150
|
+
await (0, promises_2.pipeline)((0, fs_1.createReadStream)(uncompressedFileName), (0, zlib_1.createGzip)(), (0, fs_1.createWriteStream)(compressedFileName));
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
throw new Error(`Could not compress file: ${err.message}`, { cause: err });
|
|
154
|
+
}
|
|
169
155
|
};
|
|
170
|
-
|
|
171
156
|
/**
|
|
172
157
|
* Extract a .gz file and save it to a specified location
|
|
173
158
|
*
|
|
@@ -176,546 +161,409 @@ const gzipFile = async (uncompressedFileName, compressedFileName) => {
|
|
|
176
161
|
* @return {Promise} A promise that resolves when the file is unzipped
|
|
177
162
|
*/
|
|
178
163
|
const unzipFile = async (inputFilename, outputFilename) => {
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
164
|
+
const mimeType = await detectCompressedMimeType(inputFilename);
|
|
165
|
+
const extractFunctions = {
|
|
166
|
+
'application/gzip': {
|
|
167
|
+
extractor: zlib_1.createGunzip,
|
|
168
|
+
},
|
|
169
|
+
};
|
|
170
|
+
const extractionInfo = extractFunctions[mimeType];
|
|
171
|
+
if (!extractionInfo) {
|
|
172
|
+
throw new Error(`unsupported file format: ${mimeType}`);
|
|
173
|
+
}
|
|
174
|
+
const source = (0, fs_1.createReadStream)(inputFilename);
|
|
175
|
+
const destination = (0, fs_1.createWriteStream)(outputFilename);
|
|
176
|
+
await (0, promises_2.pipeline)(source, extractionInfo.extractor(), destination);
|
|
192
177
|
};
|
|
193
178
|
exports.unzipFile = unzipFile;
|
|
194
179
|
async function getFileMeta(fileName) {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
isCompressed
|
|
208
|
-
};
|
|
180
|
+
const fileSize = await getFileSize(fileName);
|
|
181
|
+
const basename = path_1.default.basename(fileName);
|
|
182
|
+
// TODO Validate File basename... encodeURIComponent, maybe...?
|
|
183
|
+
const mimeType = await detectCompressedMimeType(fileName);
|
|
184
|
+
// TODO Only allow a subset of Mime Types...?
|
|
185
|
+
const isCompressed = ['application/zip', 'application/gzip'].includes(mimeType);
|
|
186
|
+
return {
|
|
187
|
+
basename,
|
|
188
|
+
fileName,
|
|
189
|
+
fileSize,
|
|
190
|
+
isCompressed,
|
|
191
|
+
};
|
|
209
192
|
}
|
|
210
|
-
async function uploadImportFileToS3({
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
debug(`
|
|
237
|
-
await
|
|
238
|
-
|
|
239
|
-
fileMeta.fileSize
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
debug(`Calculated file ${hashType} checksum: ${checksum}\n`);
|
|
248
|
-
const result = fileMeta.fileSize < MULTIPART_THRESHOLD ? await uploadUsingPutObject({
|
|
249
|
-
app,
|
|
250
|
-
env,
|
|
251
|
-
fileMeta,
|
|
252
|
-
progressCallback
|
|
253
|
-
}) : await uploadUsingMultipart({
|
|
254
|
-
app,
|
|
255
|
-
env,
|
|
256
|
-
fileMeta,
|
|
257
|
-
progressCallback
|
|
258
|
-
});
|
|
259
|
-
return {
|
|
260
|
-
fileMeta,
|
|
261
|
-
checksum,
|
|
262
|
-
result
|
|
263
|
-
};
|
|
193
|
+
async function uploadImportFileToS3({ app, env, fileMeta, progressCallback, hashType = 'md5', }) {
|
|
194
|
+
let tmpDir;
|
|
195
|
+
try {
|
|
196
|
+
tmpDir = await getWorkingTempDir();
|
|
197
|
+
}
|
|
198
|
+
catch (err) {
|
|
199
|
+
throw new Error(`Unable to create temporary working directory: ${err.message}`, { cause: err });
|
|
200
|
+
}
|
|
201
|
+
debug(`File ${chalk_1.default.cyan(fileMeta.basename)} is ~ ${Math.floor(fileMeta.fileSize / file_size_1.MB_IN_BYTES)} MB\n`);
|
|
202
|
+
// TODO Compression will probably fail over a certain file size... break into pieces...?
|
|
203
|
+
// TODO if needed add a flag to bypass auto-compression
|
|
204
|
+
if (!fileMeta.isCompressed && fileMeta.fileSize >= exports.COMPRESS_THRESHOLD) {
|
|
205
|
+
// Compress to the temp dir & annotate `fileMeta`
|
|
206
|
+
const uncompressedFileName = fileMeta.fileName;
|
|
207
|
+
const uncompressedFileSize = fileMeta.fileSize;
|
|
208
|
+
fileMeta.basename = fileMeta.basename.replace(/(.gz)?$/i, '.gz');
|
|
209
|
+
fileMeta.fileName = path_1.default.join(tmpDir, fileMeta.basename);
|
|
210
|
+
debug(`Compressing the file to ${chalk_1.default.cyan(fileMeta.fileName)} prior to transfer...`);
|
|
211
|
+
await gzipFile(uncompressedFileName, fileMeta.fileName);
|
|
212
|
+
fileMeta.isCompressed = true;
|
|
213
|
+
fileMeta.fileSize = await getFileSize(fileMeta.fileName);
|
|
214
|
+
debug(`Compressed file is ~ ${Math.floor(fileMeta.fileSize / file_size_1.MB_IN_BYTES)} MB\n`);
|
|
215
|
+
const fewerBytes = uncompressedFileSize - fileMeta.fileSize;
|
|
216
|
+
const calculation = `${(fewerBytes / file_size_1.MB_IN_BYTES).toFixed(2)}MB (${Math.floor((100 * fewerBytes) / uncompressedFileSize)}%)`;
|
|
217
|
+
debug(`** Compression resulted in a ${calculation} smaller file 📦 **\n`);
|
|
218
|
+
}
|
|
219
|
+
debug(`Calculating file ${hashType} checksum...`);
|
|
220
|
+
const checksum = await (0, exports.getFileHash)(fileMeta.fileName, hashType);
|
|
221
|
+
debug(`Calculated file ${hashType} checksum: ${checksum}\n`);
|
|
222
|
+
const result = fileMeta.fileSize < exports.MULTIPART_THRESHOLD
|
|
223
|
+
? await uploadUsingPutObject({ app, env, fileMeta, progressCallback })
|
|
224
|
+
: await uploadUsingMultipart({ app, env, fileMeta, progressCallback });
|
|
225
|
+
return {
|
|
226
|
+
fileMeta,
|
|
227
|
+
checksum,
|
|
228
|
+
result,
|
|
229
|
+
};
|
|
264
230
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
env,
|
|
273
|
-
fileMeta: {
|
|
274
|
-
basename,
|
|
275
|
-
fileContent,
|
|
276
|
-
fileName,
|
|
277
|
-
fileSize
|
|
278
|
-
},
|
|
279
|
-
progressCallback
|
|
280
|
-
}) {
|
|
281
|
-
debug(`Uploading ${_chalk.default.cyan(basename)} to S3 using the \`PutObject\` command`);
|
|
282
|
-
const presignedRequest = await getSignedUploadRequestData({
|
|
283
|
-
appId: app.id,
|
|
284
|
-
envId: env.id,
|
|
285
|
-
basename,
|
|
286
|
-
action: 'PutObject'
|
|
287
|
-
});
|
|
288
|
-
const fetchOptions = presignedRequest.options;
|
|
289
|
-
fetchOptions.headers = {
|
|
290
|
-
...fetchOptions.headers,
|
|
291
|
-
'Content-Length': `${fileSize}` // This has to be a string
|
|
292
|
-
};
|
|
293
|
-
|
|
294
|
-
// Build a fresh request body for every attempt. The upload streams the file
|
|
295
|
-
// from disk, and a stream can only be consumed once; recreating it per
|
|
296
|
-
// attempt lets `fetchWithRetry` retry without reusing a consumed stream
|
|
297
|
-
// (which would otherwise throw `... should not be disturbed or locked`).
|
|
298
|
-
const createBody = () => {
|
|
299
|
-
if (fileContent) {
|
|
300
|
-
return fileContent;
|
|
301
|
-
}
|
|
302
|
-
let readBytes = 0;
|
|
303
|
-
const progressTransform = createProgressTransform(data => {
|
|
304
|
-
readBytes += data.length;
|
|
305
|
-
const percentage = `${Math.floor(100 * readBytes / fileSize)}%`;
|
|
306
|
-
debug(percentage);
|
|
307
|
-
if (typeof progressCallback === 'function') {
|
|
308
|
-
progressCallback(percentage);
|
|
309
|
-
}
|
|
231
|
+
async function uploadUsingPutObject({ app, env, fileMeta: { basename, fileContent, fileName, fileSize }, progressCallback, }) {
|
|
232
|
+
debug(`Uploading ${chalk_1.default.cyan(basename)} to S3 using the \`PutObject\` command`);
|
|
233
|
+
const presignedRequest = await getSignedUploadRequestData({
|
|
234
|
+
appId: app.id,
|
|
235
|
+
envId: env.id,
|
|
236
|
+
basename,
|
|
237
|
+
action: 'PutObject',
|
|
310
238
|
});
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
239
|
+
const fetchOptions = presignedRequest.options;
|
|
240
|
+
fetchOptions.headers = {
|
|
241
|
+
...fetchOptions.headers,
|
|
242
|
+
'Content-Length': `${fileSize}`, // This has to be a string
|
|
243
|
+
};
|
|
244
|
+
// Build a fresh request body for every attempt. The upload streams the file
|
|
245
|
+
// from disk, and a stream can only be consumed once; recreating it per
|
|
246
|
+
// attempt lets `fetchWithRetry` retry without reusing a consumed stream
|
|
247
|
+
// (which would otherwise throw `... should not be disturbed or locked`).
|
|
248
|
+
const createBody = () => {
|
|
249
|
+
if (fileContent) {
|
|
250
|
+
return fileContent;
|
|
251
|
+
}
|
|
252
|
+
let readBytes = 0;
|
|
253
|
+
const progressTransform = createProgressTransform(data => {
|
|
254
|
+
readBytes += data.length;
|
|
255
|
+
const percentage = `${Math.floor((100 * readBytes) / fileSize)}%`;
|
|
256
|
+
debug(percentage);
|
|
257
|
+
if (typeof progressCallback === 'function') {
|
|
258
|
+
progressCallback(percentage);
|
|
259
|
+
}
|
|
260
|
+
});
|
|
261
|
+
return (0, fs_1.createReadStream)(fileName).pipe(progressTransform);
|
|
262
|
+
};
|
|
263
|
+
const response = await fetchWithRetry(presignedRequest.url, fetchOptions, 3, createBody);
|
|
264
|
+
if (response.status === 200) {
|
|
265
|
+
return 'ok';
|
|
266
|
+
}
|
|
267
|
+
const result = await response.text();
|
|
268
|
+
// TODO is any additional hardening needed here?
|
|
269
|
+
const parser = new xml2js_1.Parser({
|
|
270
|
+
explicitArray: false,
|
|
271
|
+
ignoreAttrs: true,
|
|
330
272
|
});
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
({
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
Code
|
|
341
|
-
Message
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
273
|
+
let parsedResponse;
|
|
274
|
+
try {
|
|
275
|
+
parsedResponse = (await parser.parseStringPromise(result));
|
|
276
|
+
}
|
|
277
|
+
catch (err) {
|
|
278
|
+
throw new Error(`Invalid response from cloud service. ${err.message}`, {
|
|
279
|
+
cause: err,
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
let Code;
|
|
283
|
+
let Message;
|
|
284
|
+
if (parsedResponse?.Error) {
|
|
285
|
+
({ Code, Message } = parsedResponse.Error);
|
|
286
|
+
}
|
|
287
|
+
else {
|
|
288
|
+
Code = `HTTP Error ${response.status}`;
|
|
289
|
+
Message = response.statusText;
|
|
290
|
+
}
|
|
291
|
+
throw new Error(`Unable to upload to cloud storage. ${JSON.stringify({ Code, Message })}`);
|
|
347
292
|
}
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
debug({
|
|
393
|
-
uploadId
|
|
394
|
-
});
|
|
395
|
-
const parts = getPartBoundaries(fileMeta.fileSize);
|
|
396
|
-
const etagResults = await uploadParts({
|
|
397
|
-
app,
|
|
398
|
-
env,
|
|
399
|
-
fileMeta,
|
|
400
|
-
parts,
|
|
401
|
-
uploadId,
|
|
402
|
-
progressCallback
|
|
403
|
-
});
|
|
404
|
-
debug({
|
|
405
|
-
etagResults
|
|
406
|
-
});
|
|
407
|
-
return completeMultipartUpload({
|
|
408
|
-
app,
|
|
409
|
-
env,
|
|
410
|
-
basename,
|
|
411
|
-
uploadId,
|
|
412
|
-
etagResults
|
|
413
|
-
});
|
|
293
|
+
async function uploadUsingMultipart({ app, env, fileMeta, progressCallback, }) {
|
|
294
|
+
const { basename } = fileMeta;
|
|
295
|
+
debug(`Uploading ${chalk_1.default.cyan(basename)} to S3 using the Multipart API.`);
|
|
296
|
+
const presignedCreateMultipartUpload = await getSignedUploadRequestData({
|
|
297
|
+
appId: app.id,
|
|
298
|
+
envId: env.id,
|
|
299
|
+
basename,
|
|
300
|
+
action: 'CreateMultipartUpload',
|
|
301
|
+
});
|
|
302
|
+
const multipartUploadResponse = await fetchWithRetry(presignedCreateMultipartUpload.url, presignedCreateMultipartUpload.options);
|
|
303
|
+
const multipartUploadResult = await multipartUploadResponse.text();
|
|
304
|
+
// TODO is any hardening needed here?
|
|
305
|
+
const parser = new xml2js_1.Parser({
|
|
306
|
+
explicitArray: false,
|
|
307
|
+
ignoreAttrs: true,
|
|
308
|
+
});
|
|
309
|
+
const parsedResponse = (await parser.parseStringPromise(multipartUploadResult));
|
|
310
|
+
if ('Error' in parsedResponse) {
|
|
311
|
+
const { Code, Message } = parsedResponse.Error;
|
|
312
|
+
throw new Error(`Unable to create cloud storage object. Error: ${JSON.stringify({ Code, Message })}`);
|
|
313
|
+
}
|
|
314
|
+
if (!('InitiateMultipartUploadResult' in parsedResponse) ||
|
|
315
|
+
!parsedResponse.InitiateMultipartUploadResult.UploadId) {
|
|
316
|
+
throw new Error(`Unable to get Upload ID from cloud storage. Error: ${multipartUploadResult}`);
|
|
317
|
+
}
|
|
318
|
+
const uploadId = parsedResponse.InitiateMultipartUploadResult.UploadId;
|
|
319
|
+
debug({ uploadId });
|
|
320
|
+
const parts = getPartBoundaries(fileMeta.fileSize);
|
|
321
|
+
const etagResults = await uploadParts({
|
|
322
|
+
app,
|
|
323
|
+
env,
|
|
324
|
+
fileMeta,
|
|
325
|
+
parts,
|
|
326
|
+
uploadId,
|
|
327
|
+
progressCallback,
|
|
328
|
+
});
|
|
329
|
+
debug({ etagResults });
|
|
330
|
+
return completeMultipartUpload({
|
|
331
|
+
app,
|
|
332
|
+
env,
|
|
333
|
+
basename,
|
|
334
|
+
uploadId,
|
|
335
|
+
etagResults,
|
|
336
|
+
});
|
|
414
337
|
}
|
|
415
|
-
async function getSignedUploadRequestData({
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
etagResults,
|
|
421
|
-
uploadId = undefined,
|
|
422
|
-
partNumber = undefined
|
|
423
|
-
}, retries = SIGNED_REQUEST_RETRIES) {
|
|
424
|
-
const WPVIP_DEPLOY_TOKEN = process.env.WPVIP_DEPLOY_TOKEN;
|
|
425
|
-
const reqOptions = {
|
|
426
|
-
method: 'POST',
|
|
427
|
-
body: {
|
|
428
|
-
action,
|
|
429
|
-
appId,
|
|
430
|
-
basename,
|
|
431
|
-
envId,
|
|
432
|
-
etagResults,
|
|
433
|
-
partNumber,
|
|
434
|
-
uploadId
|
|
435
|
-
}
|
|
436
|
-
};
|
|
437
|
-
if (WPVIP_DEPLOY_TOKEN) {
|
|
438
|
-
reqOptions.headers = {
|
|
439
|
-
Authorization: `Bearer ${WPVIP_DEPLOY_TOKEN}`
|
|
338
|
+
async function getSignedUploadRequestData({ action, appId, basename, envId, etagResults, uploadId = undefined, partNumber = undefined, }, retries = SIGNED_REQUEST_RETRIES) {
|
|
339
|
+
const WPVIP_DEPLOY_TOKEN = process.env.WPVIP_DEPLOY_TOKEN;
|
|
340
|
+
const reqOptions = {
|
|
341
|
+
method: 'POST',
|
|
342
|
+
body: { action, appId, basename, envId, etagResults, partNumber, uploadId },
|
|
440
343
|
};
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
344
|
+
if (WPVIP_DEPLOY_TOKEN) {
|
|
345
|
+
reqOptions.headers = {
|
|
346
|
+
Authorization: `Bearer ${WPVIP_DEPLOY_TOKEN}`,
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
// A large upload makes one of these requests per part, so a single network blip
|
|
350
|
+
// would otherwise abort the whole upload. The JSON body is re-serialized on every
|
|
351
|
+
// call, so retrying is safe.
|
|
352
|
+
for (let attempt = 0;; attempt++) {
|
|
353
|
+
const isLastAttempt = attempt >= retries;
|
|
354
|
+
let response;
|
|
355
|
+
try {
|
|
356
|
+
// eslint-disable-next-line no-await-in-loop
|
|
357
|
+
response = await (0, http_1.default)('/upload/site-import-presigned-url', reqOptions);
|
|
358
|
+
if (response.status === 200) {
|
|
359
|
+
// Body transport failures must stay inside the retry boundary.
|
|
360
|
+
// eslint-disable-next-line no-await-in-loop
|
|
361
|
+
return (await response.json());
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
catch (err) {
|
|
365
|
+
if (err instanceof SyntaxError || isLastAttempt) {
|
|
366
|
+
throw withCauseMessage(err);
|
|
367
|
+
}
|
|
368
|
+
response = undefined;
|
|
369
|
+
}
|
|
370
|
+
const isRetryableStatus = response && (response.status === 429 || response.status >= 500);
|
|
371
|
+
if (response && (!isRetryableStatus || isLastAttempt)) {
|
|
372
|
+
// eslint-disable-next-line no-await-in-loop
|
|
373
|
+
throw new Error((await response.text()) || response.statusText);
|
|
374
|
+
}
|
|
375
|
+
// eslint-disable-next-line no-await-in-loop
|
|
376
|
+
await response?.body?.cancel().catch(err => debug('Response cleanup failed: %s', err));
|
|
453
377
|
// eslint-disable-next-line no-await-in-loop
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
} catch (err) {
|
|
457
|
-
if (err instanceof SyntaxError || isLastAttempt) {
|
|
458
|
-
throw withCauseMessage(err);
|
|
459
|
-
}
|
|
460
|
-
response = undefined;
|
|
461
|
-
}
|
|
462
|
-
const isRetryableStatus = response && (response.status === 429 || response.status >= 500);
|
|
463
|
-
if (response && (!isRetryableStatus || isLastAttempt)) {
|
|
464
|
-
// eslint-disable-next-line no-await-in-loop
|
|
465
|
-
throw new Error((await response.text()) || response.statusText);
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
// eslint-disable-next-line no-await-in-loop
|
|
469
|
-
await response?.body?.cancel().catch(err => debug('Response cleanup failed: %s', err));
|
|
470
|
-
// eslint-disable-next-line no-await-in-loop
|
|
471
|
-
await (0, _promises3.setTimeout)(Math.pow(2, attempt) * 1000); // 1000, 2000, 4000
|
|
472
|
-
}
|
|
378
|
+
await (0, promises_3.setTimeout)(Math.pow(2, attempt) * 1000); // 1000, 2000, 4000
|
|
379
|
+
}
|
|
473
380
|
}
|
|
474
381
|
async function checkFileAccess(fileName) {
|
|
475
|
-
|
|
382
|
+
return (0, promises_1.access)(fileName, fs_1.constants.R_OK);
|
|
476
383
|
}
|
|
477
384
|
async function isFile(fileName) {
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
385
|
+
try {
|
|
386
|
+
const stats = await (0, promises_1.stat)(fileName);
|
|
387
|
+
return stats.isFile();
|
|
388
|
+
}
|
|
389
|
+
catch (err) {
|
|
390
|
+
debug(`isFile error: ${err.message}`);
|
|
391
|
+
return false;
|
|
392
|
+
}
|
|
485
393
|
}
|
|
486
394
|
async function getFileSize(fileName) {
|
|
487
|
-
|
|
488
|
-
|
|
395
|
+
const stats = await (0, promises_1.stat)(fileName);
|
|
396
|
+
return stats.size;
|
|
489
397
|
}
|
|
490
398
|
async function detectCompressedMimeType(fileName) {
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
buffer
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
return '
|
|
504
|
-
}
|
|
505
|
-
return '';
|
|
399
|
+
const ZIP_MAGIC_NUMBER = '504b0304';
|
|
400
|
+
const GZ_MAGIC_NUMBER = '1f8b';
|
|
401
|
+
const file = await (0, promises_1.open)(fileName, 'r');
|
|
402
|
+
const { buffer } = await file.read(Buffer.alloc(4), 0, 4, 0);
|
|
403
|
+
const fileHeader = buffer.toString('hex');
|
|
404
|
+
await file.close();
|
|
405
|
+
if (ZIP_MAGIC_NUMBER === fileHeader.slice(0, ZIP_MAGIC_NUMBER.length)) {
|
|
406
|
+
return 'application/zip';
|
|
407
|
+
}
|
|
408
|
+
if (GZ_MAGIC_NUMBER === fileHeader.slice(0, GZ_MAGIC_NUMBER.length)) {
|
|
409
|
+
return 'application/gzip';
|
|
410
|
+
}
|
|
411
|
+
return '';
|
|
506
412
|
}
|
|
507
413
|
function getPartBoundaries(fileSize) {
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
414
|
+
if (fileSize < 1) {
|
|
415
|
+
throw new Error('fileSize must be greater than zero');
|
|
416
|
+
}
|
|
417
|
+
const numParts = Math.ceil(fileSize / UPLOAD_PART_SIZE);
|
|
418
|
+
return new Array(numParts).fill(undefined).map((_numPart, index) => {
|
|
419
|
+
const start = index * UPLOAD_PART_SIZE;
|
|
420
|
+
const remaining = fileSize - start;
|
|
421
|
+
const end = (remaining > UPLOAD_PART_SIZE ? start + UPLOAD_PART_SIZE : start + remaining) - 1;
|
|
422
|
+
const partSize = end + 1 - start;
|
|
423
|
+
return { end, index, partSize, start };
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
async function uploadParts({ app, env, fileMeta, uploadId, parts, progressCallback, }) {
|
|
427
|
+
let uploadsInProgress = 0;
|
|
428
|
+
const partPercentages = new Array(parts.length).fill(0);
|
|
429
|
+
const partBytesRead = new Array(parts.length).fill(0);
|
|
430
|
+
const readyForPartUpload = () => new Promise(resolve => {
|
|
431
|
+
const canDoInterval = (0, node_timers_1.setInterval)(() => {
|
|
432
|
+
if (uploadsInProgress < MAX_CONCURRENT_PART_UPLOADS) {
|
|
433
|
+
uploadsInProgress++;
|
|
434
|
+
(0, node_timers_1.clearInterval)(canDoInterval);
|
|
435
|
+
resolve();
|
|
436
|
+
}
|
|
437
|
+
}, 300);
|
|
438
|
+
});
|
|
439
|
+
const updateProgress = () => {
|
|
440
|
+
const totalBytesRead = partBytesRead.reduce((sum, bytes) => sum + bytes, 0);
|
|
441
|
+
const percentage = `${Math.floor((100 * totalBytesRead) / fileMeta.fileSize)}%`;
|
|
442
|
+
if (typeof progressCallback === 'function') {
|
|
443
|
+
progressCallback(percentage);
|
|
444
|
+
}
|
|
445
|
+
debug(partPercentages
|
|
446
|
+
.map((partPercentage, index) => {
|
|
447
|
+
const { partSize } = parts[index];
|
|
448
|
+
return `Part # ${index}: ${partPercentage}% of ${(partSize / file_size_1.MB_IN_BYTES).toFixed(2)}MB`;
|
|
449
|
+
})
|
|
450
|
+
.join('\n') +
|
|
451
|
+
`\n\nOverall Progress: ${percentage}% of ${(fileMeta.fileSize / file_size_1.MB_IN_BYTES).toFixed(2)}MB`);
|
|
522
452
|
};
|
|
523
|
-
|
|
453
|
+
const updateProgressInterval = (0, node_timers_1.setInterval)(updateProgress, 500);
|
|
454
|
+
const allDone = await Promise.all(parts.map(async (part) => {
|
|
455
|
+
const { index, partSize, start, end } = part;
|
|
456
|
+
// Build a fresh request body for every attempt. A stream can only be
|
|
457
|
+
// consumed once, so recreating the ranged read stream (and its progress
|
|
458
|
+
// PassThrough) per attempt lets `fetchWithRetry` retry without reusing a
|
|
459
|
+
// disturbed/locked stream. Reset this part's progress counters so a retry
|
|
460
|
+
// re-streams the part without double-counting bytes.
|
|
461
|
+
const createBody = () => {
|
|
462
|
+
partBytesRead[index] = 0;
|
|
463
|
+
partPercentages[index] = 0;
|
|
464
|
+
const progressTransform = createProgressTransform(data => {
|
|
465
|
+
partBytesRead[index] += data.length;
|
|
466
|
+
partPercentages[index] = Math.floor((100 * partBytesRead[index]) / partSize);
|
|
467
|
+
});
|
|
468
|
+
return (0, fs_1.createReadStream)(fileMeta.fileName, { start, end }).pipe(progressTransform);
|
|
469
|
+
};
|
|
470
|
+
await readyForPartUpload();
|
|
471
|
+
const uploadResult = await uploadPart({
|
|
472
|
+
app,
|
|
473
|
+
env,
|
|
474
|
+
fileMeta,
|
|
475
|
+
part,
|
|
476
|
+
createBody,
|
|
477
|
+
uploadId,
|
|
478
|
+
});
|
|
479
|
+
uploadsInProgress--;
|
|
480
|
+
return uploadResult;
|
|
481
|
+
}));
|
|
482
|
+
(0, node_timers_1.clearInterval)(updateProgressInterval);
|
|
483
|
+
updateProgress();
|
|
484
|
+
return allDone;
|
|
524
485
|
}
|
|
525
|
-
async function
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
// re-streams the part without double-counting bytes.
|
|
572
|
-
const createBody = () => {
|
|
573
|
-
partBytesRead[index] = 0;
|
|
574
|
-
partPercentages[index] = 0;
|
|
575
|
-
const progressTransform = createProgressTransform(data => {
|
|
576
|
-
partBytesRead[index] += data.length;
|
|
577
|
-
partPercentages[index] = Math.floor(100 * partBytesRead[index] / partSize);
|
|
578
|
-
});
|
|
579
|
-
return (0, _fs.createReadStream)(fileMeta.fileName, {
|
|
580
|
-
start,
|
|
581
|
-
end
|
|
582
|
-
}).pipe(progressTransform);
|
|
486
|
+
async function uploadPart({ app, env, fileMeta: { basename }, part, createBody, uploadId, }) {
|
|
487
|
+
const { index, partSize } = part;
|
|
488
|
+
const s3PartNumber = index + 1; // S3 multipart is indexed from 1
|
|
489
|
+
// TODO: handle failures / retries, etc.
|
|
490
|
+
const doUpload = async () => {
|
|
491
|
+
// Get the signed request data from Parker
|
|
492
|
+
const partUploadRequestData = await getSignedUploadRequestData({
|
|
493
|
+
action: 'UploadPart',
|
|
494
|
+
appId: app.id,
|
|
495
|
+
envId: env.id,
|
|
496
|
+
basename,
|
|
497
|
+
partNumber: s3PartNumber,
|
|
498
|
+
uploadId,
|
|
499
|
+
});
|
|
500
|
+
const fetchOptions = partUploadRequestData.options;
|
|
501
|
+
fetchOptions.headers = {
|
|
502
|
+
...fetchOptions.headers,
|
|
503
|
+
'Content-Length': `${partSize}`, // This has to be a string
|
|
504
|
+
/**
|
|
505
|
+
* TODO? 'Content-MD5': Buffer.from( ... ).toString( 'base64' ),
|
|
506
|
+
* Content-MD5 has to be base64 encoded.
|
|
507
|
+
* It's the hash of the entire request object & has to be included in the signature,
|
|
508
|
+
* ...so it may not be feasible to include with presigned requests.
|
|
509
|
+
*/
|
|
510
|
+
};
|
|
511
|
+
const fetchResponse = await fetchWithRetry(partUploadRequestData.url, fetchOptions, 3, createBody);
|
|
512
|
+
if (fetchResponse.status === 200) {
|
|
513
|
+
const etag = fetchResponse.headers.get('etag');
|
|
514
|
+
if (!etag) {
|
|
515
|
+
throw new Error('Unable to upload file part. Missing ETag response header.');
|
|
516
|
+
}
|
|
517
|
+
return parseEtagHeader(etag);
|
|
518
|
+
}
|
|
519
|
+
const result = await fetchResponse.text();
|
|
520
|
+
// TODO is any hardening needed here?
|
|
521
|
+
const parser = new xml2js_1.Parser({
|
|
522
|
+
explicitArray: false,
|
|
523
|
+
ignoreAttrs: true,
|
|
524
|
+
});
|
|
525
|
+
const parsed = (await parser.parseStringPromise(result));
|
|
526
|
+
const { Code, Message } = parsed.Error;
|
|
527
|
+
throw new Error(`Unable to upload file part. Error: ${JSON.stringify({ Code, Message })}`);
|
|
528
|
+
};
|
|
529
|
+
return {
|
|
530
|
+
ETag: await doUpload(),
|
|
531
|
+
PartNumber: s3PartNumber,
|
|
583
532
|
};
|
|
584
|
-
await readyForPartUpload();
|
|
585
|
-
const uploadResult = await uploadPart({
|
|
586
|
-
app,
|
|
587
|
-
env,
|
|
588
|
-
fileMeta,
|
|
589
|
-
part,
|
|
590
|
-
createBody,
|
|
591
|
-
uploadId
|
|
592
|
-
});
|
|
593
|
-
uploadsInProgress--;
|
|
594
|
-
return uploadResult;
|
|
595
|
-
}));
|
|
596
|
-
(0, _nodeTimers.clearInterval)(updateProgressInterval);
|
|
597
|
-
updateProgress();
|
|
598
|
-
return allDone;
|
|
599
533
|
}
|
|
600
|
-
async function
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
uploadId
|
|
609
|
-
}) {
|
|
610
|
-
const {
|
|
611
|
-
index,
|
|
612
|
-
partSize
|
|
613
|
-
} = part;
|
|
614
|
-
const s3PartNumber = index + 1; // S3 multipart is indexed from 1
|
|
615
|
-
|
|
616
|
-
// TODO: handle failures / retries, etc.
|
|
617
|
-
const doUpload = async () => {
|
|
618
|
-
// Get the signed request data from Parker
|
|
619
|
-
const partUploadRequestData = await getSignedUploadRequestData({
|
|
620
|
-
action: 'UploadPart',
|
|
621
|
-
appId: app.id,
|
|
622
|
-
envId: env.id,
|
|
623
|
-
basename,
|
|
624
|
-
partNumber: s3PartNumber,
|
|
625
|
-
uploadId
|
|
534
|
+
async function completeMultipartUpload({ app, env, basename, uploadId, etagResults, }) {
|
|
535
|
+
const completeMultipartUploadRequestData = await getSignedUploadRequestData({
|
|
536
|
+
action: 'CompleteMultipartUpload',
|
|
537
|
+
appId: app.id,
|
|
538
|
+
envId: env.id,
|
|
539
|
+
basename,
|
|
540
|
+
uploadId,
|
|
541
|
+
etagResults,
|
|
626
542
|
});
|
|
627
|
-
const
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
const result = await fetchResponse.text();
|
|
647
|
-
|
|
648
|
-
// TODO is any hardening needed here?
|
|
649
|
-
const parser = new _xml2js.Parser({
|
|
650
|
-
explicitArray: false,
|
|
651
|
-
ignoreAttrs: true
|
|
543
|
+
const completeMultipartUploadResponse = await fetchWithRetry(completeMultipartUploadRequestData.url, completeMultipartUploadRequestData.options);
|
|
544
|
+
if (completeMultipartUploadResponse.status !== 200) {
|
|
545
|
+
// eslint-disable-next-line @typescript-eslint/only-throw-error
|
|
546
|
+
throw await completeMultipartUploadResponse.text();
|
|
547
|
+
}
|
|
548
|
+
/**
|
|
549
|
+
* Processing of a Complete Multipart Upload request could take several minutes to complete.
|
|
550
|
+
* After Amazon S3 begins processing the request, it sends an HTTP response header that specifies a 200 OK response.
|
|
551
|
+
* While processing is in progress, Amazon S3 periodically sends white space characters to keep the connection from timing out.
|
|
552
|
+
* Because a request could fail after the initial 200 OK response has been sent, it is important that you check the
|
|
553
|
+
* response body to determine whether the request succeeded.
|
|
554
|
+
* Note that if CompleteMultipartUpload fails, applications should be prepared to retry the failed requests.
|
|
555
|
+
*
|
|
556
|
+
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
|
|
557
|
+
*/
|
|
558
|
+
const result = await completeMultipartUploadResponse.text();
|
|
559
|
+
const parser = new xml2js_1.Parser({
|
|
560
|
+
explicitArray: false,
|
|
561
|
+
ignoreAttrs: true,
|
|
652
562
|
});
|
|
653
|
-
const parsed = await parser.parseStringPromise(result);
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
}
|
|
658
|
-
|
|
659
|
-
Code,
|
|
660
|
-
Message
|
|
661
|
-
})}`);
|
|
662
|
-
};
|
|
663
|
-
return {
|
|
664
|
-
ETag: await doUpload(),
|
|
665
|
-
PartNumber: s3PartNumber
|
|
666
|
-
};
|
|
563
|
+
const parsed = (await parser.parseStringPromise(result));
|
|
564
|
+
if ('Error' in parsed) {
|
|
565
|
+
const { Code, Message } = parsed.Error;
|
|
566
|
+
throw new Error(`Unable to complete the upload. Error: ${JSON.stringify({ Code, Message })}`);
|
|
567
|
+
}
|
|
568
|
+
return parsed;
|
|
667
569
|
}
|
|
668
|
-
|
|
669
|
-
/**
|
|
670
|
-
* @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_ResponseSyntax
|
|
671
|
-
*/
|
|
672
|
-
|
|
673
|
-
async function completeMultipartUpload({
|
|
674
|
-
app,
|
|
675
|
-
env,
|
|
676
|
-
basename,
|
|
677
|
-
uploadId,
|
|
678
|
-
etagResults
|
|
679
|
-
}) {
|
|
680
|
-
const completeMultipartUploadRequestData = await getSignedUploadRequestData({
|
|
681
|
-
action: 'CompleteMultipartUpload',
|
|
682
|
-
appId: app.id,
|
|
683
|
-
envId: env.id,
|
|
684
|
-
basename,
|
|
685
|
-
uploadId,
|
|
686
|
-
etagResults
|
|
687
|
-
});
|
|
688
|
-
const completeMultipartUploadResponse = await fetchWithRetry(completeMultipartUploadRequestData.url, completeMultipartUploadRequestData.options);
|
|
689
|
-
if (completeMultipartUploadResponse.status !== 200) {
|
|
690
|
-
// eslint-disable-next-line @typescript-eslint/only-throw-error
|
|
691
|
-
throw await completeMultipartUploadResponse.text();
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
/**
|
|
695
|
-
* Processing of a Complete Multipart Upload request could take several minutes to complete.
|
|
696
|
-
* After Amazon S3 begins processing the request, it sends an HTTP response header that specifies a 200 OK response.
|
|
697
|
-
* While processing is in progress, Amazon S3 periodically sends white space characters to keep the connection from timing out.
|
|
698
|
-
* Because a request could fail after the initial 200 OK response has been sent, it is important that you check the
|
|
699
|
-
* response body to determine whether the request succeeded.
|
|
700
|
-
* Note that if CompleteMultipartUpload fails, applications should be prepared to retry the failed requests.
|
|
701
|
-
*
|
|
702
|
-
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
|
|
703
|
-
*/
|
|
704
|
-
const result = await completeMultipartUploadResponse.text();
|
|
705
|
-
const parser = new _xml2js.Parser({
|
|
706
|
-
explicitArray: false,
|
|
707
|
-
ignoreAttrs: true
|
|
708
|
-
});
|
|
709
|
-
const parsed = await parser.parseStringPromise(result);
|
|
710
|
-
if ('Error' in parsed) {
|
|
711
|
-
const {
|
|
712
|
-
Code,
|
|
713
|
-
Message
|
|
714
|
-
} = parsed.Error;
|
|
715
|
-
throw new Error(`Unable to complete the upload. Error: ${JSON.stringify({
|
|
716
|
-
Code,
|
|
717
|
-
Message
|
|
718
|
-
})}`);
|
|
719
|
-
}
|
|
720
|
-
return parsed;
|
|
721
|
-
}
|