@automattic/vip 2.12.0 → 2.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -1
- package/dist/bin/vip-cache-purge-url.js +93 -0
- package/{package/dist/bin/vip-config.js → dist/bin/vip-cache.js} +6 -3
- package/dist/bin/vip-dev-env-import-sql.js +8 -1
- package/dist/bin/vip-dev-env-start.js +2 -1
- package/dist/bin/vip-dev-env-update.js +1 -0
- package/dist/bin/vip.js +1 -1
- package/dist/lib/api/cache-purge.js +76 -0
- package/dist/lib/dev-environment/dev-environment-lando.js +31 -3
- package/dist/lib/envvar/read-file.js +2 -18
- package/{package/dist/lib/envvar → dist/lib}/read-file.js +3 -6
- package/dist/lib/validations/sql.js +134 -52
- package/npm-shrinkwrap.json +79 -79
- package/package.json +4 -2
- package/package/dist/bin/vip-app-list.js +0 -73
- package/package/dist/bin/vip-app.js +0 -76
- package/package/dist/bin/vip-config-envvar-delete.js +0 -97
- package/package/dist/bin/vip-config-envvar-get-all.js +0 -94
- package/package/dist/bin/vip-config-envvar-get.js +0 -79
- package/package/dist/bin/vip-config-envvar-list.js +0 -91
- package/package/dist/bin/vip-config-envvar-set.js +0 -123
- package/package/dist/bin/vip-config-envvar.js +0 -23
- package/package/dist/bin/vip-dev-env-create.js +0 -105
- package/package/dist/bin/vip-dev-env-destroy.js +0 -56
- package/package/dist/bin/vip-dev-env-exec.js +0 -67
- package/package/dist/bin/vip-dev-env-import-media.js +0 -51
- package/package/dist/bin/vip-dev-env-import-sql.js +0 -83
- package/package/dist/bin/vip-dev-env-import.js +0 -32
- package/package/dist/bin/vip-dev-env-info.js +0 -61
- package/package/dist/bin/vip-dev-env-list.js +0 -46
- package/package/dist/bin/vip-dev-env-start.js +0 -77
- package/package/dist/bin/vip-dev-env-stop.js +0 -52
- package/package/dist/bin/vip-dev-env-update.js +0 -89
- package/package/dist/bin/vip-dev-env.js +0 -23
- package/package/dist/bin/vip-import-media-abort.js +0 -132
- package/package/dist/bin/vip-import-media-status.js +0 -84
- package/package/dist/bin/vip-import-media.js +0 -168
- package/package/dist/bin/vip-import-sql-status.js +0 -83
- package/package/dist/bin/vip-import-sql.js +0 -580
- package/package/dist/bin/vip-import-validate-files.js +0 -191
- package/package/dist/bin/vip-import-validate-sql.js +0 -34
- package/package/dist/bin/vip-import.js +0 -20
- package/package/dist/bin/vip-logs.js +0 -232
- package/package/dist/bin/vip-search-replace.js +0 -71
- package/package/dist/bin/vip-sync.js +0 -191
- package/package/dist/bin/vip-whoami.js +0 -67
- package/package/dist/bin/vip-wp.js +0 -555
- package/package/dist/bin/vip.js +0 -149
- package/package/dist/lib/analytics/clients/client.js +0 -1
- package/package/dist/lib/analytics/clients/pendo.js +0 -92
- package/package/dist/lib/analytics/clients/stub.js +0 -19
- package/package/dist/lib/analytics/clients/tracks.js +0 -128
- package/package/dist/lib/analytics/index.js +0 -45
- package/package/dist/lib/api/app.js +0 -70
- package/package/dist/lib/api/feature-flags.js +0 -39
- package/package/dist/lib/api/user.js +0 -58
- package/package/dist/lib/api.js +0 -136
- package/package/dist/lib/app-logs/app-logs.js +0 -70
- package/package/dist/lib/cli/apiConfig.js +0 -90
- package/package/dist/lib/cli/command.js +0 -606
- package/package/dist/lib/cli/envAlias.js +0 -60
- package/package/dist/lib/cli/exit.js +0 -33
- package/package/dist/lib/cli/format.js +0 -213
- package/package/dist/lib/cli/pager.js +0 -52
- package/package/dist/lib/cli/progress.js +0 -208
- package/package/dist/lib/cli/prompt.js +0 -37
- package/package/dist/lib/cli/repo.js +0 -77
- package/package/dist/lib/client-file-uploader.js +0 -602
- package/package/dist/lib/constants/dev-environment.js +0 -42
- package/package/dist/lib/constants/file-size.js +0 -14
- package/package/dist/lib/dev-environment/dev-environment-cli.js +0 -508
- package/package/dist/lib/dev-environment/dev-environment-core.js +0 -620
- package/package/dist/lib/dev-environment/dev-environment-lando.js +0 -330
- package/package/dist/lib/dev-environment/types.js +0 -1
- package/package/dist/lib/env.js +0 -36
- package/package/dist/lib/envvar/api-delete.js +0 -56
- package/package/dist/lib/envvar/api-get-all.js +0 -59
- package/package/dist/lib/envvar/api-get.js +0 -24
- package/package/dist/lib/envvar/api-list.js +0 -60
- package/package/dist/lib/envvar/api-set.js +0 -58
- package/package/dist/lib/envvar/api.js +0 -104
- package/package/dist/lib/envvar/input.js +0 -55
- package/package/dist/lib/envvar/logging.js +0 -33
- package/package/dist/lib/http/socks-proxy-agent.js +0 -25
- package/package/dist/lib/keychain/browser.js +0 -35
- package/package/dist/lib/keychain/insecure.js +0 -63
- package/package/dist/lib/keychain/keychain.js +0 -1
- package/package/dist/lib/keychain/secure.js +0 -36
- package/package/dist/lib/keychain.js +0 -36
- package/package/dist/lib/media-import/media-file-import.js +0 -34
- package/package/dist/lib/media-import/progress.js +0 -86
- package/package/dist/lib/media-import/status.js +0 -335
- package/package/dist/lib/rollbar.js +0 -35
- package/package/dist/lib/search-and-replace.js +0 -203
- package/package/dist/lib/site-import/db-file-import.js +0 -46
- package/package/dist/lib/site-import/status.js +0 -444
- package/package/dist/lib/token.js +0 -132
- package/package/dist/lib/tracker.js +0 -96
- package/package/dist/lib/validations/is-multi-site-sql-dump.js +0 -59
- package/package/dist/lib/validations/is-multi-site.js +0 -99
- package/package/dist/lib/validations/line-by-line.js +0 -92
- package/package/dist/lib/validations/site-type.js +0 -66
- package/package/dist/lib/validations/sql.js +0 -371
- package/package/dist/lib/vip-import-validate-files.js +0 -548
- package/package/vip.iml +0 -11
|
@@ -1,602 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.getFileMeta = getFileMeta;
|
|
7
|
-
exports.uploadImportSqlFileToS3 = uploadImportSqlFileToS3;
|
|
8
|
-
exports.uploadUsingPutObject = uploadUsingPutObject;
|
|
9
|
-
exports.uploadUsingMultipart = uploadUsingMultipart;
|
|
10
|
-
exports.getSignedUploadRequestData = getSignedUploadRequestData;
|
|
11
|
-
exports.checkFileAccess = checkFileAccess;
|
|
12
|
-
exports.getFileStats = getFileStats;
|
|
13
|
-
exports.isFile = isFile;
|
|
14
|
-
exports.getFileSize = getFileSize;
|
|
15
|
-
exports.detectCompressedMimeType = detectCompressedMimeType;
|
|
16
|
-
exports.getPartBoundaries = getPartBoundaries;
|
|
17
|
-
exports.uploadParts = uploadParts;
|
|
18
|
-
exports.uploadPart = uploadPart;
|
|
19
|
-
exports.completeMultipartUpload = completeMultipartUpload;
|
|
20
|
-
exports.gzipFile = exports.getFileMD5Hash = exports.getWorkingTempDir = exports.MULTIPART_THRESHOLD = exports.COMPRESS_THRESHOLD = void 0;
|
|
21
|
-
|
|
22
|
-
var _fs = _interopRequireWildcard(require("fs"));
|
|
23
|
-
|
|
24
|
-
var _os = _interopRequireDefault(require("os"));
|
|
25
|
-
|
|
26
|
-
var _path = _interopRequireDefault(require("path"));
|
|
27
|
-
|
|
28
|
-
var _nodeFetch = _interopRequireDefault(require("node-fetch"));
|
|
29
|
-
|
|
30
|
-
var _chalk = _interopRequireDefault(require("chalk"));
|
|
31
|
-
|
|
32
|
-
var _zlib = require("zlib");
|
|
33
|
-
|
|
34
|
-
var _crypto = require("crypto");
|
|
35
|
-
|
|
36
|
-
var _stream = require("stream");
|
|
37
|
-
|
|
38
|
-
var _xml2js = require("xml2js");
|
|
39
|
-
|
|
40
|
-
var _debug = _interopRequireDefault(require("debug"));
|
|
41
|
-
|
|
42
|
-
var _api = _interopRequireDefault(require("./api"));
|
|
43
|
-
|
|
44
|
-
var _fileSize = require("./constants/file-size");
|
|
45
|
-
|
|
46
|
-
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
47
|
-
|
|
48
|
-
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
|
|
49
|
-
|
|
50
|
-
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
|
|
51
|
-
|
|
52
|
-
/**
|
|
53
|
-
*
|
|
54
|
-
* @format
|
|
55
|
-
*/
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* External dependencies
|
|
59
|
-
*/
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Internal dependencies
|
|
63
|
-
*/
|
|
64
|
-
const debug = (0, _debug.default)('vip:lib/client-file-uploader'); // Files smaller than COMPRESS_THRESHOLD will not be compressed before upload
|
|
65
|
-
|
|
66
|
-
const COMPRESS_THRESHOLD = 16 * _fileSize.MB_IN_BYTES; // Files smaller than MULTIPART_THRESHOLD will use `PutObject` vs Multipart Uploads
|
|
67
|
-
|
|
68
|
-
exports.COMPRESS_THRESHOLD = COMPRESS_THRESHOLD;
|
|
69
|
-
const MULTIPART_THRESHOLD = 32 * _fileSize.MB_IN_BYTES; // This is how big each part of a Multipart Upload is (except the last / remainder)
|
|
70
|
-
|
|
71
|
-
exports.MULTIPART_THRESHOLD = MULTIPART_THRESHOLD;
|
|
72
|
-
const UPLOAD_PART_SIZE = 16 * _fileSize.MB_IN_BYTES; // How many parts will upload at the same time
|
|
73
|
-
|
|
74
|
-
const MAX_CONCURRENT_PART_UPLOADS = 5;
|
|
75
|
-
|
|
76
|
-
const getWorkingTempDir = async () => new Promise((resolve, reject) => {
|
|
77
|
-
_fs.default.mkdtemp(_path.default.join(_os.default.tmpdir(), 'vip-client-file-uploader'), (err, dir) => {
|
|
78
|
-
if (err) {
|
|
79
|
-
return reject(err);
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
resolve(dir);
|
|
83
|
-
});
|
|
84
|
-
});
|
|
85
|
-
|
|
86
|
-
exports.getWorkingTempDir = getWorkingTempDir;
|
|
87
|
-
|
|
88
|
-
const getFileMD5Hash = async fileName => new Promise((resolve, reject) => _fs.default.createReadStream(fileName).pipe((0, _crypto.createHash)('md5').setEncoding('hex')).on('finish', function () {
|
|
89
|
-
resolve(this.read());
|
|
90
|
-
}).on('error', error => reject(`could not generate file hash: ${error}`)));
|
|
91
|
-
|
|
92
|
-
exports.getFileMD5Hash = getFileMD5Hash;
|
|
93
|
-
|
|
94
|
-
const gzipFile = async (uncompressedFileName, compressedFileName) => new Promise((resolve, reject) => _fs.default.createReadStream(uncompressedFileName).pipe((0, _zlib.createGzip)()).pipe(_fs.default.createWriteStream(compressedFileName)).on('finish', resolve).on('error', error => reject(`could not compress file: ${error}`)));
|
|
95
|
-
|
|
96
|
-
exports.gzipFile = gzipFile;
|
|
97
|
-
|
|
98
|
-
async function getFileMeta(fileName) {
|
|
99
|
-
return new Promise(async resolve => {
|
|
100
|
-
const fileSize = await getFileSize(fileName);
|
|
101
|
-
|
|
102
|
-
const basename = _path.default.basename(fileName); // TODO Validate File basename... encodeURIComponent, maybe...?
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
const mimeType = await detectCompressedMimeType(fileName); // TODO Only allow a subset of Mime Types...?
|
|
106
|
-
|
|
107
|
-
const isCompressed = ['application/zip', 'application/gzip'].includes(mimeType);
|
|
108
|
-
resolve({
|
|
109
|
-
basename,
|
|
110
|
-
fileName,
|
|
111
|
-
fileSize,
|
|
112
|
-
isCompressed
|
|
113
|
-
});
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
async function uploadImportSqlFileToS3({
|
|
118
|
-
app,
|
|
119
|
-
env,
|
|
120
|
-
fileMeta,
|
|
121
|
-
progressCallback
|
|
122
|
-
}) {
|
|
123
|
-
let tmpDir;
|
|
124
|
-
|
|
125
|
-
try {
|
|
126
|
-
tmpDir = await getWorkingTempDir();
|
|
127
|
-
} catch (err) {
|
|
128
|
-
throw `Unable to create temporary working directory: ${err}`;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
debug(`File ${_chalk.default.cyan(fileMeta.basename)} is ~ ${Math.floor(fileMeta.fileSize / _fileSize.MB_IN_BYTES)} MB\n`); // TODO Compression will probably fail over a certain file size... break into pieces...?
|
|
132
|
-
// TODO if needed add a flag to bypass auto-compression
|
|
133
|
-
|
|
134
|
-
if (!fileMeta.isCompressed && fileMeta.fileSize >= COMPRESS_THRESHOLD) {
|
|
135
|
-
// Compress to the temp dir & annotate `fileMeta`
|
|
136
|
-
const uncompressedFileName = fileMeta.fileName;
|
|
137
|
-
const uncompressedFileSize = fileMeta.fileSize;
|
|
138
|
-
fileMeta.basename = fileMeta.basename.replace(/(.gz)?$/i, '.gz');
|
|
139
|
-
fileMeta.fileName = _path.default.join(tmpDir, fileMeta.basename);
|
|
140
|
-
debug(`Compressing the file to ${_chalk.default.cyan(fileMeta.fileName)} prior to transfer...`);
|
|
141
|
-
await gzipFile(uncompressedFileName, fileMeta.fileName);
|
|
142
|
-
fileMeta.isCompressed = true;
|
|
143
|
-
fileMeta.fileSize = await getFileSize(fileMeta.fileName);
|
|
144
|
-
debug(`Compressed file is ~ ${Math.floor(fileMeta.fileSize / _fileSize.MB_IN_BYTES)} MB\n`);
|
|
145
|
-
const fewerBytes = uncompressedFileSize - fileMeta.fileSize;
|
|
146
|
-
const calculation = `${(fewerBytes / _fileSize.MB_IN_BYTES).toFixed(2)}MB (${Math.floor(100 * fewerBytes / uncompressedFileSize)}%)`;
|
|
147
|
-
debug(`** Compression resulted in a ${calculation} smaller file 📦 **\n`);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
debug('Calculating file md5 checksum...');
|
|
151
|
-
const md5 = await getFileMD5Hash(fileMeta.fileName);
|
|
152
|
-
debug(`Calculated file md5 checksum: ${md5}\n`);
|
|
153
|
-
const result = fileMeta.fileSize < MULTIPART_THRESHOLD ? await uploadUsingPutObject({
|
|
154
|
-
app,
|
|
155
|
-
env,
|
|
156
|
-
fileMeta,
|
|
157
|
-
progressCallback
|
|
158
|
-
}) : await uploadUsingMultipart({
|
|
159
|
-
app,
|
|
160
|
-
env,
|
|
161
|
-
fileMeta,
|
|
162
|
-
progressCallback
|
|
163
|
-
});
|
|
164
|
-
return {
|
|
165
|
-
fileMeta,
|
|
166
|
-
md5,
|
|
167
|
-
result
|
|
168
|
-
};
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
async function uploadUsingPutObject({
|
|
172
|
-
app,
|
|
173
|
-
env,
|
|
174
|
-
fileMeta: {
|
|
175
|
-
basename,
|
|
176
|
-
fileContent,
|
|
177
|
-
fileName,
|
|
178
|
-
fileSize
|
|
179
|
-
},
|
|
180
|
-
progressCallback
|
|
181
|
-
}) {
|
|
182
|
-
debug(`Uploading ${_chalk.default.cyan(basename)} to S3 using the \`PutObject\` command`);
|
|
183
|
-
const presignedRequest = await getSignedUploadRequestData({
|
|
184
|
-
appId: app.id,
|
|
185
|
-
envId: env.id,
|
|
186
|
-
basename,
|
|
187
|
-
action: 'PutObject'
|
|
188
|
-
});
|
|
189
|
-
const fetchOptions = presignedRequest.options;
|
|
190
|
-
fetchOptions.headers = { ...fetchOptions.headers,
|
|
191
|
-
'Content-Length': `${fileSize}` // This has to be a string
|
|
192
|
-
|
|
193
|
-
};
|
|
194
|
-
let readBytes = 0;
|
|
195
|
-
const progressPassThrough = new _stream.PassThrough();
|
|
196
|
-
progressPassThrough.on('data', data => {
|
|
197
|
-
readBytes += data.length;
|
|
198
|
-
const percentage = Math.floor(100 * readBytes / fileSize) + '%';
|
|
199
|
-
debug(percentage);
|
|
200
|
-
|
|
201
|
-
if (typeof progressCallback === 'function') {
|
|
202
|
-
progressCallback(percentage);
|
|
203
|
-
}
|
|
204
|
-
});
|
|
205
|
-
const response = await (0, _nodeFetch.default)(presignedRequest.url, { ...fetchOptions,
|
|
206
|
-
body: fileContent ? fileContent : _fs.default.createReadStream(fileName).pipe(progressPassThrough)
|
|
207
|
-
});
|
|
208
|
-
|
|
209
|
-
if (response.status === 200) {
|
|
210
|
-
return 'ok';
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
const result = await response.text(); // TODO is any additional hardening needed here?
|
|
214
|
-
|
|
215
|
-
const parser = new _xml2js.Parser({
|
|
216
|
-
explicitArray: false,
|
|
217
|
-
ignoreAttrs: true
|
|
218
|
-
});
|
|
219
|
-
let parsedResponse;
|
|
220
|
-
|
|
221
|
-
try {
|
|
222
|
-
parsedResponse = await parser.parseStringPromise(result);
|
|
223
|
-
} catch (err) {
|
|
224
|
-
throw `Invalid response from cloud service. ${err}`;
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
const {
|
|
228
|
-
Code,
|
|
229
|
-
Message
|
|
230
|
-
} = parsedResponse.Error || {};
|
|
231
|
-
throw `Unable to upload to cloud storage. ${JSON.stringify({
|
|
232
|
-
Code,
|
|
233
|
-
Message
|
|
234
|
-
})}`;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
async function uploadUsingMultipart({
|
|
238
|
-
app,
|
|
239
|
-
env,
|
|
240
|
-
fileMeta,
|
|
241
|
-
progressCallback
|
|
242
|
-
}) {
|
|
243
|
-
const {
|
|
244
|
-
basename
|
|
245
|
-
} = fileMeta;
|
|
246
|
-
debug(`Uploading ${_chalk.default.cyan(basename)} to S3 using the Multipart API.`);
|
|
247
|
-
const presignedCreateMultipartUpload = await getSignedUploadRequestData({
|
|
248
|
-
appId: app.id,
|
|
249
|
-
envId: env.id,
|
|
250
|
-
basename,
|
|
251
|
-
action: 'CreateMultipartUpload'
|
|
252
|
-
});
|
|
253
|
-
const multipartUploadResponse = await (0, _nodeFetch.default)(presignedCreateMultipartUpload.url, presignedCreateMultipartUpload.options);
|
|
254
|
-
const multipartUploadResult = await multipartUploadResponse.text(); // TODO is any hardening needed here?
|
|
255
|
-
|
|
256
|
-
const parser = new _xml2js.Parser({
|
|
257
|
-
explicitArray: false,
|
|
258
|
-
ignoreAttrs: true
|
|
259
|
-
});
|
|
260
|
-
const parsedResponse = await parser.parseStringPromise(multipartUploadResult);
|
|
261
|
-
|
|
262
|
-
if (parsedResponse.Error) {
|
|
263
|
-
const {
|
|
264
|
-
Code,
|
|
265
|
-
Message
|
|
266
|
-
} = parsedResponse.Error;
|
|
267
|
-
throw `Unable to create cloud storage object. Error: ${JSON.stringify({
|
|
268
|
-
Code,
|
|
269
|
-
Message
|
|
270
|
-
})}`;
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
if (!parsedResponse && parsedResponse.InitiateMultipartUploadResult && parsedResponse.InitiateMultipartUploadResult.UploadId) {
|
|
274
|
-
throw `Unable to get Upload ID from cloud storage. Error: ${multipartUploadResult}`;
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
const uploadId = parsedResponse.InitiateMultipartUploadResult.UploadId;
|
|
278
|
-
debug({
|
|
279
|
-
uploadId
|
|
280
|
-
});
|
|
281
|
-
const parts = getPartBoundaries(fileMeta.fileSize);
|
|
282
|
-
const etagResults = await uploadParts({
|
|
283
|
-
app,
|
|
284
|
-
env,
|
|
285
|
-
fileMeta,
|
|
286
|
-
parts,
|
|
287
|
-
uploadId,
|
|
288
|
-
progressCallback
|
|
289
|
-
});
|
|
290
|
-
debug({
|
|
291
|
-
etagResults
|
|
292
|
-
});
|
|
293
|
-
return completeMultipartUpload({
|
|
294
|
-
app,
|
|
295
|
-
env,
|
|
296
|
-
basename,
|
|
297
|
-
uploadId,
|
|
298
|
-
etagResults
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
async function getSignedUploadRequestData({
|
|
303
|
-
action,
|
|
304
|
-
appId,
|
|
305
|
-
basename,
|
|
306
|
-
envId,
|
|
307
|
-
etagResults,
|
|
308
|
-
uploadId = undefined,
|
|
309
|
-
partNumber = undefined
|
|
310
|
-
}) {
|
|
311
|
-
const {
|
|
312
|
-
apiFetch
|
|
313
|
-
} = await (0, _api.default)();
|
|
314
|
-
const response = await apiFetch('/upload/site-import-presigned-url', {
|
|
315
|
-
method: 'POST',
|
|
316
|
-
body: {
|
|
317
|
-
action,
|
|
318
|
-
appId,
|
|
319
|
-
basename,
|
|
320
|
-
envId,
|
|
321
|
-
etagResults,
|
|
322
|
-
partNumber,
|
|
323
|
-
uploadId
|
|
324
|
-
}
|
|
325
|
-
});
|
|
326
|
-
|
|
327
|
-
if (response.status !== 200) {
|
|
328
|
-
throw (await response.text()) || response.statusText;
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
return response.json();
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
async function checkFileAccess(fileName) {
|
|
335
|
-
return _fs.default.promises.access(fileName, _fs.default.R_OK);
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
async function getFileStats(fileName) {
|
|
339
|
-
return _fs.default.promises.stat(fileName);
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
async function isFile(fileName) {
|
|
343
|
-
try {
|
|
344
|
-
const stats = await getFileStats(fileName);
|
|
345
|
-
return stats.isFile();
|
|
346
|
-
} catch (err) {
|
|
347
|
-
debug(`isFile error: ${err}`);
|
|
348
|
-
return false;
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
async function getFileSize(fileName) {
|
|
353
|
-
const stats = await getFileStats(fileName);
|
|
354
|
-
return stats.size;
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
async function detectCompressedMimeType(fileName) {
|
|
358
|
-
const ZIP_MAGIC_NUMBER = '504b0304';
|
|
359
|
-
const GZ_MAGIC_NUMBER = '1f8b';
|
|
360
|
-
let fileHeader = '';
|
|
361
|
-
return new Promise(resolve => {
|
|
362
|
-
_fs.default.createReadStream(fileName, {
|
|
363
|
-
start: 0,
|
|
364
|
-
end: 8,
|
|
365
|
-
encoding: 'hex'
|
|
366
|
-
}).on('data', data => {
|
|
367
|
-
fileHeader += data;
|
|
368
|
-
}).on('end', () => {
|
|
369
|
-
if (ZIP_MAGIC_NUMBER === fileHeader.slice(0, ZIP_MAGIC_NUMBER.length)) {
|
|
370
|
-
return resolve('application/zip');
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
if (GZ_MAGIC_NUMBER === fileHeader.slice(0, GZ_MAGIC_NUMBER.length)) {
|
|
374
|
-
return resolve('application/gzip');
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
resolve();
|
|
378
|
-
});
|
|
379
|
-
});
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
function getPartBoundaries(fileSize) {
|
|
383
|
-
if (fileSize < 1) {
|
|
384
|
-
throw 'fileSize must be greater than zero';
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
const numParts = Math.ceil(fileSize / UPLOAD_PART_SIZE);
|
|
388
|
-
return new Array(numParts).fill(undefined).map((_numPart, index) => {
|
|
389
|
-
const start = index * UPLOAD_PART_SIZE;
|
|
390
|
-
const remaining = fileSize - start;
|
|
391
|
-
const end = (remaining > UPLOAD_PART_SIZE ? start + UPLOAD_PART_SIZE : start + remaining) - 1;
|
|
392
|
-
const partSize = end + 1 - start;
|
|
393
|
-
return {
|
|
394
|
-
end,
|
|
395
|
-
index,
|
|
396
|
-
partSize,
|
|
397
|
-
start
|
|
398
|
-
};
|
|
399
|
-
});
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
async function uploadParts({
|
|
403
|
-
app,
|
|
404
|
-
env,
|
|
405
|
-
fileMeta,
|
|
406
|
-
uploadId,
|
|
407
|
-
parts,
|
|
408
|
-
progressCallback
|
|
409
|
-
}) {
|
|
410
|
-
let uploadsInProgress = 0;
|
|
411
|
-
let totalBytesRead = 0;
|
|
412
|
-
const partPercentages = new Array(parts.length).fill(0);
|
|
413
|
-
|
|
414
|
-
const readyForPartUpload = () => new Promise(resolve => {
|
|
415
|
-
const canDoInterval = setInterval(() => {
|
|
416
|
-
if (uploadsInProgress < MAX_CONCURRENT_PART_UPLOADS) {
|
|
417
|
-
uploadsInProgress++;
|
|
418
|
-
clearInterval(canDoInterval);
|
|
419
|
-
resolve();
|
|
420
|
-
}
|
|
421
|
-
}, 300);
|
|
422
|
-
});
|
|
423
|
-
|
|
424
|
-
const updateProgress = () => {
|
|
425
|
-
const percentage = Math.floor(100 * totalBytesRead / fileMeta.fileSize) + '%';
|
|
426
|
-
|
|
427
|
-
if (typeof progressCallback === 'function') {
|
|
428
|
-
progressCallback(percentage);
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
debug(partPercentages.map((partPercentage, index) => {
|
|
432
|
-
const {
|
|
433
|
-
partSize
|
|
434
|
-
} = parts[index];
|
|
435
|
-
return `Part # ${index}: ${partPercentage}% of ${(partSize / _fileSize.MB_IN_BYTES).toFixed(2)}MB`;
|
|
436
|
-
}).join('\n') + `\n\nOverall Progress: ${percentage}% of ${(fileMeta.fileSize / _fileSize.MB_IN_BYTES).toFixed(2)}MB`);
|
|
437
|
-
};
|
|
438
|
-
|
|
439
|
-
const updateProgressInterval = setInterval(updateProgress, 500);
|
|
440
|
-
const allDone = await Promise.all(parts.map(async part => {
|
|
441
|
-
const {
|
|
442
|
-
index,
|
|
443
|
-
partSize
|
|
444
|
-
} = part;
|
|
445
|
-
const progressPassThrough = new _stream.PassThrough();
|
|
446
|
-
let partBytesRead = 0;
|
|
447
|
-
progressPassThrough.on('data', data => {
|
|
448
|
-
totalBytesRead += data.length;
|
|
449
|
-
partBytesRead += data.length;
|
|
450
|
-
partPercentages[index] = Math.floor(100 * partBytesRead / partSize);
|
|
451
|
-
});
|
|
452
|
-
await readyForPartUpload();
|
|
453
|
-
const uploadResult = await uploadPart({
|
|
454
|
-
app,
|
|
455
|
-
env,
|
|
456
|
-
fileMeta,
|
|
457
|
-
part,
|
|
458
|
-
progressPassThrough,
|
|
459
|
-
uploadId
|
|
460
|
-
});
|
|
461
|
-
uploadsInProgress--;
|
|
462
|
-
return uploadResult;
|
|
463
|
-
}));
|
|
464
|
-
clearInterval(updateProgressInterval);
|
|
465
|
-
updateProgress();
|
|
466
|
-
return allDone;
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
async function uploadPart({
|
|
470
|
-
app,
|
|
471
|
-
env,
|
|
472
|
-
fileMeta: {
|
|
473
|
-
basename,
|
|
474
|
-
fileName
|
|
475
|
-
},
|
|
476
|
-
part,
|
|
477
|
-
progressPassThrough,
|
|
478
|
-
uploadId
|
|
479
|
-
}) {
|
|
480
|
-
const {
|
|
481
|
-
end,
|
|
482
|
-
index,
|
|
483
|
-
partSize,
|
|
484
|
-
start
|
|
485
|
-
} = part;
|
|
486
|
-
const s3PartNumber = index + 1; // S3 multipart is indexed from 1
|
|
487
|
-
// TODO: handle failures / retries, etc.
|
|
488
|
-
|
|
489
|
-
const doUpload = async () => {
|
|
490
|
-
// Get the signed request data from Parker
|
|
491
|
-
const partUploadRequestData = await getSignedUploadRequestData({
|
|
492
|
-
action: 'UploadPart',
|
|
493
|
-
appId: app.id,
|
|
494
|
-
envId: env.id,
|
|
495
|
-
basename,
|
|
496
|
-
partNumber: s3PartNumber,
|
|
497
|
-
uploadId
|
|
498
|
-
});
|
|
499
|
-
const fetchOptions = partUploadRequestData.options;
|
|
500
|
-
fetchOptions.headers = { ...fetchOptions.headers,
|
|
501
|
-
'Content-Length': `${partSize}` // This has to be a string
|
|
502
|
-
|
|
503
|
-
/**
|
|
504
|
-
* TODO? 'Content-MD5': Buffer.from( ... ).toString( 'base64' ),
|
|
505
|
-
* Content-MD5 has to be base64 encoded.
|
|
506
|
-
* It's the hash of the entire request object & has to be included in the signature,
|
|
507
|
-
* ...so it may not be feasible to include with presigned requests.
|
|
508
|
-
*/
|
|
509
|
-
|
|
510
|
-
};
|
|
511
|
-
fetchOptions.body = _fs.default.createReadStream(fileName, {
|
|
512
|
-
start,
|
|
513
|
-
end
|
|
514
|
-
}).pipe(progressPassThrough);
|
|
515
|
-
const fetchResponse = await (0, _nodeFetch.default)(partUploadRequestData.url, fetchOptions);
|
|
516
|
-
|
|
517
|
-
if (fetchResponse.status === 200) {
|
|
518
|
-
const responseHeaders = fetchResponse.headers.raw();
|
|
519
|
-
const [etag] = responseHeaders.etag;
|
|
520
|
-
return JSON.parse(etag);
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
const result = await fetchResponse.text(); // TODO is any hardening needed here?
|
|
524
|
-
|
|
525
|
-
const parser = new _xml2js.Parser({
|
|
526
|
-
explicitArray: false,
|
|
527
|
-
ignoreAttrs: true
|
|
528
|
-
});
|
|
529
|
-
const parsed = await parser.parseStringPromise(result);
|
|
530
|
-
|
|
531
|
-
if (parsed.Error) {
|
|
532
|
-
const {
|
|
533
|
-
Code,
|
|
534
|
-
Message
|
|
535
|
-
} = parsed.Error;
|
|
536
|
-
throw `Unable to upload file part. Error: ${JSON.stringify({
|
|
537
|
-
Code,
|
|
538
|
-
Message
|
|
539
|
-
})}`;
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
return parsed;
|
|
543
|
-
};
|
|
544
|
-
|
|
545
|
-
return {
|
|
546
|
-
ETag: await doUpload(),
|
|
547
|
-
PartNumber: s3PartNumber
|
|
548
|
-
};
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
async function completeMultipartUpload({
|
|
552
|
-
app,
|
|
553
|
-
env,
|
|
554
|
-
basename,
|
|
555
|
-
uploadId,
|
|
556
|
-
etagResults
|
|
557
|
-
}) {
|
|
558
|
-
const completeMultipartUploadRequestData = await getSignedUploadRequestData({
|
|
559
|
-
action: 'CompleteMultipartUpload',
|
|
560
|
-
appId: app.id,
|
|
561
|
-
envId: env.id,
|
|
562
|
-
basename,
|
|
563
|
-
uploadId,
|
|
564
|
-
etagResults
|
|
565
|
-
});
|
|
566
|
-
const completeMultipartUploadResponse = await (0, _nodeFetch.default)(completeMultipartUploadRequestData.url, completeMultipartUploadRequestData.options);
|
|
567
|
-
|
|
568
|
-
if (completeMultipartUploadResponse.status !== 200) {
|
|
569
|
-
throw await completeMultipartUploadResponse.text();
|
|
570
|
-
}
|
|
571
|
-
/**
|
|
572
|
-
* Processing of a Complete Multipart Upload request could take several minutes to complete.
|
|
573
|
-
* After Amazon S3 begins processing the request, it sends an HTTP response header that specifies a 200 OK response.
|
|
574
|
-
* While processing is in progress, Amazon S3 periodically sends white space characters to keep the connection from timing out.
|
|
575
|
-
* Because a request could fail after the initial 200 OK response has been sent, it is important that you check the
|
|
576
|
-
* response body to determine whether the request succeeded.
|
|
577
|
-
* Note that if CompleteMultipartUpload fails, applications should be prepared to retry the failed requests.
|
|
578
|
-
*
|
|
579
|
-
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
|
|
580
|
-
*/
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
const result = await completeMultipartUploadResponse.text();
|
|
584
|
-
const parser = new _xml2js.Parser({
|
|
585
|
-
explicitArray: false,
|
|
586
|
-
ignoreAttrs: true
|
|
587
|
-
});
|
|
588
|
-
const parsed = await parser.parseStringPromise(result);
|
|
589
|
-
|
|
590
|
-
if (parsed.Error) {
|
|
591
|
-
const {
|
|
592
|
-
Code,
|
|
593
|
-
Message
|
|
594
|
-
} = parsed.Error;
|
|
595
|
-
throw `Unable to complete the upload. Error: ${JSON.stringify({
|
|
596
|
-
Code,
|
|
597
|
-
Message
|
|
598
|
-
})}`;
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
return parsed;
|
|
602
|
-
}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.DEV_ENVIRONMENT_PHP_VERSIONS = exports.DEV_ENVIRONMENT_WORDPRESS_VERSION_TTL = exports.DEV_ENVIRONMENT_WORDPRESS_CACHE_KEY = exports.DEV_ENVIRONMENT_WORDPRESS_VERSIONS_URI = exports.DEV_ENVIRONMENT_RAW_GITHUB_HOST = exports.DEV_ENVIRONMENT_COMPONENTS = exports.DEV_ENVIRONMENT_NOT_FOUND = exports.DEV_ENVIRONMENT_PROMPT_INTRO = exports.DEV_ENVIRONMENT_DEFAULTS = exports.DEV_ENVIRONMENT_FULL_COMMAND = exports.DEV_ENVIRONMENT_SUBCOMMAND = void 0;
|
|
7
|
-
const DEV_ENVIRONMENT_SUBCOMMAND = 'dev-env';
|
|
8
|
-
exports.DEV_ENVIRONMENT_SUBCOMMAND = DEV_ENVIRONMENT_SUBCOMMAND;
|
|
9
|
-
const DEV_ENVIRONMENT_FULL_COMMAND = `vip ${DEV_ENVIRONMENT_SUBCOMMAND}`;
|
|
10
|
-
exports.DEV_ENVIRONMENT_FULL_COMMAND = DEV_ENVIRONMENT_FULL_COMMAND;
|
|
11
|
-
const DEV_ENVIRONMENT_DEFAULTS = {
|
|
12
|
-
title: 'VIP Dev',
|
|
13
|
-
multisite: false,
|
|
14
|
-
elasticsearchVersion: '7.10.1',
|
|
15
|
-
mariadbVersion: '10.3',
|
|
16
|
-
phpImage: 'default'
|
|
17
|
-
};
|
|
18
|
-
exports.DEV_ENVIRONMENT_DEFAULTS = DEV_ENVIRONMENT_DEFAULTS;
|
|
19
|
-
const DEV_ENVIRONMENT_PROMPT_INTRO = 'This is a wizard to help you set up your local dev environment.\n\n' + 'Sensible default values were pre-selected for convenience. ' + 'You may also choose to create multiple environments with different settings using the --slug option.\n\n';
|
|
20
|
-
exports.DEV_ENVIRONMENT_PROMPT_INTRO = DEV_ENVIRONMENT_PROMPT_INTRO;
|
|
21
|
-
const DEV_ENVIRONMENT_NOT_FOUND = 'Environment not found.';
|
|
22
|
-
exports.DEV_ENVIRONMENT_NOT_FOUND = DEV_ENVIRONMENT_NOT_FOUND;
|
|
23
|
-
const DEV_ENVIRONMENT_COMPONENTS = ['wordpress', 'muPlugins', 'clientCode'];
|
|
24
|
-
exports.DEV_ENVIRONMENT_COMPONENTS = DEV_ENVIRONMENT_COMPONENTS;
|
|
25
|
-
const DEV_ENVIRONMENT_RAW_GITHUB_HOST = 'raw.githubusercontent.com';
|
|
26
|
-
exports.DEV_ENVIRONMENT_RAW_GITHUB_HOST = DEV_ENVIRONMENT_RAW_GITHUB_HOST;
|
|
27
|
-
const DEV_ENVIRONMENT_WORDPRESS_VERSIONS_URI = '/Automattic/vip-container-images/master/wordpress/versions.json';
|
|
28
|
-
exports.DEV_ENVIRONMENT_WORDPRESS_VERSIONS_URI = DEV_ENVIRONMENT_WORDPRESS_VERSIONS_URI;
|
|
29
|
-
const DEV_ENVIRONMENT_WORDPRESS_CACHE_KEY = 'wordpress-versions.json';
|
|
30
|
-
exports.DEV_ENVIRONMENT_WORDPRESS_CACHE_KEY = DEV_ENVIRONMENT_WORDPRESS_CACHE_KEY;
|
|
31
|
-
const DEV_ENVIRONMENT_WORDPRESS_VERSION_TTL = 86400; // once per day
|
|
32
|
-
|
|
33
|
-
exports.DEV_ENVIRONMENT_WORDPRESS_VERSION_TTL = DEV_ENVIRONMENT_WORDPRESS_VERSION_TTL;
|
|
34
|
-
const DEV_ENVIRONMENT_PHP_VERSIONS = {
|
|
35
|
-
default: 'ghcr.io/automattic/vip-container-images/php-fpm:7.4',
|
|
36
|
-
// eslint-disable-next-line quote-props -- flow does nit support non-string keys
|
|
37
|
-
'7.4': 'ghcr.io/automattic/vip-container-images/php-fpm-alt:7.4',
|
|
38
|
-
'8.0': 'ghcr.io/automattic/vip-container-images/php-fpm-alt:8.0',
|
|
39
|
-
// eslint-disable-next-line quote-props
|
|
40
|
-
'8.1': 'ghcr.io/automattic/vip-container-images/php-fpm-alt:8.1'
|
|
41
|
-
};
|
|
42
|
-
exports.DEV_ENVIRONMENT_PHP_VERSIONS = DEV_ENVIRONMENT_PHP_VERSIONS;
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.TB_IN_BYTES = exports.GB_IN_BYTES = exports.MB_IN_BYTES = exports.KB_IN_BYTES = void 0;
|
|
7
|
-
const KB_IN_BYTES = 1024;
|
|
8
|
-
exports.KB_IN_BYTES = KB_IN_BYTES;
|
|
9
|
-
const MB_IN_BYTES = 1024 * KB_IN_BYTES;
|
|
10
|
-
exports.MB_IN_BYTES = MB_IN_BYTES;
|
|
11
|
-
const GB_IN_BYTES = 1024 * MB_IN_BYTES;
|
|
12
|
-
exports.GB_IN_BYTES = GB_IN_BYTES;
|
|
13
|
-
const TB_IN_BYTES = 1024 * GB_IN_BYTES;
|
|
14
|
-
exports.TB_IN_BYTES = TB_IN_BYTES;
|