@automattic/vip 3.21.1 → 3.21.3-dev.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/dev-env.lando.template.yml.ejs +4 -0
- package/dist/bin/vip-app-deploy.js +1 -1
- package/dist/bin/vip-dev-env-envvar-delete.js +5 -4
- package/dist/bin/vip-dev-env-envvar-get-all.js +6 -5
- package/dist/bin/vip-dev-env-envvar-get.js +5 -4
- package/dist/bin/vip-dev-env-envvar-list.js +4 -3
- package/dist/bin/vip-dev-env-envvar-set.js +7 -6
- package/dist/bin/vip-dev-env-envvar.js +12 -12
- package/dist/bin/vip-dev-env-sync-sql.js +1 -0
- package/dist/bin/vip-import-media.js +67 -8
- package/dist/bin/vip-import-sql.js +1 -1
- package/dist/commands/dev-env-sync-sql.js +74 -4
- package/dist/commands/export-sql.js +2 -2
- package/dist/lib/api.js +41 -2
- package/dist/lib/client-file-uploader.js +3 -2
- package/dist/lib/constants/dev-environment.js +2 -2
- package/dist/lib/live-backup-copy.js +1 -1
- package/dist/lib/media-import/utils.js +18 -0
- package/dist/lib/utils.js +10 -0
- package/npm-shrinkwrap.json +946 -1124
- package/package.json +6 -7
|
@@ -160,6 +160,10 @@ services:
|
|
|
160
160
|
services:
|
|
161
161
|
image: elasticsearch:8.18.2
|
|
162
162
|
command: /usr/local/bin/docker-entrypoint.sh
|
|
163
|
+
deploy:
|
|
164
|
+
resources:
|
|
165
|
+
limits:
|
|
166
|
+
memory: 1GB
|
|
163
167
|
environment:
|
|
164
168
|
ELASTICSEARCH_IS_DEDICATED_NODE: 'no'
|
|
165
169
|
ELASTICSEARCH_CLUSTER_NAME: 'bespin'
|
|
@@ -156,7 +156,7 @@ Processing the file for deployment to your environment...
|
|
|
156
156
|
},
|
|
157
157
|
checksum,
|
|
158
158
|
result
|
|
159
|
-
} = await (0, _clientFileUploader.
|
|
159
|
+
} = await (0, _clientFileUploader.uploadImportFileToS3)(uploadParams);
|
|
160
160
|
startDeployVariables.input = {
|
|
161
161
|
id: appId,
|
|
162
162
|
environmentId: envId,
|
|
@@ -7,12 +7,13 @@ var _devEnvironmentCli = require("../lib/dev-environment/dev-environment-cli");
|
|
|
7
7
|
var _envVars = require("../lib/dev-environment/env-vars");
|
|
8
8
|
var _logging = require("../lib/envvar/logging");
|
|
9
9
|
var _tracker = require("../lib/tracker");
|
|
10
|
+
var _utils = require("../lib/utils");
|
|
10
11
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
11
|
-
const exampleUsage = 'vip dev-env envvar delete';
|
|
12
|
-
const usage = 'vip dev-env envvar delete
|
|
12
|
+
const exampleUsage = 'vip dev-env envvar delete --slug=example-site';
|
|
13
|
+
const usage = 'vip dev-env envvar delete';
|
|
13
14
|
const examples = [{
|
|
14
15
|
usage: `${exampleUsage} MY_VARIABLE`,
|
|
15
|
-
description: 'Delete the environment variable "MY_VARIABLE" from the environment.'
|
|
16
|
+
description: 'Delete the environment variable "MY_VARIABLE" from the local environment.'
|
|
16
17
|
}];
|
|
17
18
|
async function deleteEnvVarCommand(args, opt) {
|
|
18
19
|
(0, _logging.debug)('args: %o, opt: %o', args, opt);
|
|
@@ -26,7 +27,7 @@ async function deleteEnvVarCommand(args, opt) {
|
|
|
26
27
|
const data = await (0, _envVars.readEnvFile)(slug);
|
|
27
28
|
const envVars = [];
|
|
28
29
|
data.forEach(line => {
|
|
29
|
-
const [key] =
|
|
30
|
+
const [key] = (0, _utils.splitKeyValueString)(line);
|
|
30
31
|
if (key !== name) {
|
|
31
32
|
envVars.push(line);
|
|
32
33
|
} else {
|
|
@@ -8,15 +8,16 @@ var _devEnvironmentCli = require("../lib/dev-environment/dev-environment-cli");
|
|
|
8
8
|
var _envVars = require("../lib/dev-environment/env-vars");
|
|
9
9
|
var _logging = require("../lib/envvar/logging");
|
|
10
10
|
var _tracker = require("../lib/tracker");
|
|
11
|
+
var _utils = require("../lib/utils");
|
|
11
12
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
12
|
-
const exampleUsage = 'vip dev-env envvar get-all';
|
|
13
|
-
const usage = 'vip dev-env envvar get-all
|
|
13
|
+
const exampleUsage = 'vip dev-env envvar get-all --slug=example-site';
|
|
14
|
+
const usage = 'vip dev-env envvar get-all';
|
|
14
15
|
const examples = [{
|
|
15
16
|
usage: exampleUsage,
|
|
16
|
-
description: 'Retrieve a list of all environment variables in the default table format.'
|
|
17
|
+
description: 'Retrieve a list of all local environment variables in the default table format.'
|
|
17
18
|
}, {
|
|
18
19
|
usage: `${exampleUsage} --format=csv`,
|
|
19
|
-
description: 'Retrieve a list of all environment variables in CSV format.'
|
|
20
|
+
description: 'Retrieve a list of all local environment variables in CSV format.'
|
|
20
21
|
}];
|
|
21
22
|
async function getAllEnvVarsCommand(args, opt) {
|
|
22
23
|
(0, _logging.debug)('args: %o, opt: %o', args, opt);
|
|
@@ -28,7 +29,7 @@ async function getAllEnvVarsCommand(args, opt) {
|
|
|
28
29
|
try {
|
|
29
30
|
const data = await (0, _envVars.readEnvFile)(slug);
|
|
30
31
|
const envVars = data.map(line => {
|
|
31
|
-
const [key, value] =
|
|
32
|
+
const [key, value] = (0, _utils.splitKeyValueString)(line);
|
|
32
33
|
return {
|
|
33
34
|
name: key,
|
|
34
35
|
value: (0, _envVars.parseEnvValue)(value)
|
|
@@ -7,12 +7,13 @@ var _devEnvironmentCli = require("../lib/dev-environment/dev-environment-cli");
|
|
|
7
7
|
var _envVars = require("../lib/dev-environment/env-vars");
|
|
8
8
|
var _logging = require("../lib/envvar/logging");
|
|
9
9
|
var _tracker = require("../lib/tracker");
|
|
10
|
+
var _utils = require("../lib/utils");
|
|
10
11
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
11
|
-
const exampleUsage = 'vip dev-env envvar get';
|
|
12
|
-
const usage = 'vip dev-env envvar get
|
|
12
|
+
const exampleUsage = 'vip dev-env envvar get --slug=example-site';
|
|
13
|
+
const usage = 'vip dev-env envvar get';
|
|
13
14
|
const examples = [{
|
|
14
15
|
usage: `${exampleUsage} MY_VARIABLE`,
|
|
15
|
-
description: 'Retrieve the value of the environment variable "MY_VARIABLE".'
|
|
16
|
+
description: 'Retrieve the value of the local environment variable "MY_VARIABLE".'
|
|
16
17
|
}];
|
|
17
18
|
async function getEnvVarsCommand(args, opt) {
|
|
18
19
|
(0, _logging.debug)('args: %o, opt: %o', args, opt);
|
|
@@ -23,7 +24,7 @@ async function getEnvVarsCommand(args, opt) {
|
|
|
23
24
|
await (0, _tracker.trackEvent)(`${trackingPrefix}execute`, trackingInfo);
|
|
24
25
|
try {
|
|
25
26
|
const data = await (0, _envVars.readEnvFile)(slug);
|
|
26
|
-
const envVar = data.map(line =>
|
|
27
|
+
const envVar = data.map(line => (0, _utils.splitKeyValueString)(line)).find(([key]) => name === key.trim());
|
|
27
28
|
if (undefined === envVar) {
|
|
28
29
|
process.stderr.write(_chalk.default.yellow(`The environment variable "${name}" does not exist\n`));
|
|
29
30
|
process.exitCode = 1;
|
|
@@ -8,11 +8,12 @@ var _devEnvironmentCli = require("../lib/dev-environment/dev-environment-cli");
|
|
|
8
8
|
var _envVars = require("../lib/dev-environment/env-vars");
|
|
9
9
|
var _logging = require("../lib/envvar/logging");
|
|
10
10
|
var _tracker = require("../lib/tracker");
|
|
11
|
+
var _utils = require("../lib/utils");
|
|
11
12
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
12
13
|
const usage = 'vip dev-env envvar list';
|
|
13
14
|
const examples = [{
|
|
14
|
-
usage: 'vip dev-env envvar list -
|
|
15
|
-
description: 'List the names of all environment variables.'
|
|
15
|
+
usage: 'vip dev-env envvar list --slug=example-site',
|
|
16
|
+
description: 'List the names of all environment variables on the local environment in table format.'
|
|
16
17
|
}];
|
|
17
18
|
async function listEnvVarsCommand(args, opt) {
|
|
18
19
|
(0, _logging.debug)('args: %o, opt: %o', args, opt);
|
|
@@ -24,7 +25,7 @@ async function listEnvVarsCommand(args, opt) {
|
|
|
24
25
|
try {
|
|
25
26
|
const data = await (0, _envVars.readEnvFile)(slug);
|
|
26
27
|
const envVars = data.map(line => {
|
|
27
|
-
const [key] =
|
|
28
|
+
const [key] = (0, _utils.splitKeyValueString)(line);
|
|
28
29
|
return {
|
|
29
30
|
name: key.trim()
|
|
30
31
|
};
|
|
@@ -10,18 +10,19 @@ var _input = require("../lib/envvar/input");
|
|
|
10
10
|
var _logging = require("../lib/envvar/logging");
|
|
11
11
|
var _readFile = require("../lib/envvar/read-file");
|
|
12
12
|
var _tracker = require("../lib/tracker");
|
|
13
|
+
var _utils = require("../lib/utils");
|
|
13
14
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
14
|
-
const exampleUsage = 'vip dev-env envvar set';
|
|
15
|
-
const usage = 'vip dev-env envvar set
|
|
15
|
+
const exampleUsage = 'vip dev-env envvar set --slug=example-site';
|
|
16
|
+
const usage = 'vip dev-env envvar set';
|
|
16
17
|
const examples = [{
|
|
17
18
|
usage: `${exampleUsage} MY_VARIABLE`,
|
|
18
|
-
description: 'Add or update the environment variable "MY_VARIABLE" and assign its value at the prompt.'
|
|
19
|
+
description: 'Add or update the local environment variable "MY_VARIABLE" and assign its value at the prompt.'
|
|
19
20
|
}, {
|
|
20
21
|
usage: `${exampleUsage} MY_VARIABLE MY_VALUE`,
|
|
21
|
-
description: 'Add or update the environment variable "MY_VARIABLE" and assign its value to "MY_VALUE".'
|
|
22
|
+
description: 'Add or update the local environment variable "MY_VARIABLE" and assign its value to "MY_VALUE".'
|
|
22
23
|
}, {
|
|
23
24
|
usage: `${exampleUsage} MULTILINE_ENV_VAR --from-file=envvar-value.txt`,
|
|
24
|
-
description: 'Add or update the environment variable "MULTILINE_ENV_VAR" and assign the multiline contents of local file envvar-value.txt as its value.'
|
|
25
|
+
description: 'Add or update the local environment variable "MULTILINE_ENV_VAR" and assign the multiline contents of local file envvar-value.txt as its value.'
|
|
25
26
|
}];
|
|
26
27
|
async function deleteEnvVarCommand(args, opt) {
|
|
27
28
|
(0, _logging.debug)('args: %o, opt: %o', args, opt);
|
|
@@ -51,7 +52,7 @@ async function deleteEnvVarCommand(args, opt) {
|
|
|
51
52
|
newValue = (0, _envVars.quoteEnvValue)(newValue);
|
|
52
53
|
let replaced = false;
|
|
53
54
|
const envVars = data.map(line => {
|
|
54
|
-
const [key] =
|
|
55
|
+
const [key] = (0, _utils.splitKeyValueString)(line);
|
|
55
56
|
if (key === name) {
|
|
56
57
|
replaced = true;
|
|
57
58
|
return `${key}=${newValue}`;
|
|
@@ -4,24 +4,24 @@
|
|
|
4
4
|
var _command = _interopRequireDefault(require("../lib/cli/command"));
|
|
5
5
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
6
6
|
const usage = 'vip dev-env envvar';
|
|
7
|
-
const exampleUsage = 'vip dev-env envvar
|
|
7
|
+
const exampleUsage = 'vip dev-env envvar';
|
|
8
8
|
const examples = [{
|
|
9
|
-
usage: `${exampleUsage} set MY_VARIABLE`,
|
|
10
|
-
description: 'Add or update the environment variable "MY_VARIABLE" and assign its value at the prompt.'
|
|
9
|
+
usage: `${exampleUsage} set --slug=example-site MY_VARIABLE`,
|
|
10
|
+
description: 'Add or update the local environment variable "MY_VARIABLE" and assign its value at the prompt.'
|
|
11
11
|
}, {
|
|
12
|
-
usage: `${exampleUsage} get MY_VARIABLE`,
|
|
13
|
-
description: 'Retrieve the value of the environment variable "MY_VARIABLE".'
|
|
12
|
+
usage: `${exampleUsage} get --slug=example-site MY_VARIABLE`,
|
|
13
|
+
description: 'Retrieve the value of the local environment variable "MY_VARIABLE".'
|
|
14
14
|
}, {
|
|
15
|
-
usage: `${exampleUsage} get-all`,
|
|
16
|
-
description: 'Retrieve a list of all environment variables in the default table format.'
|
|
15
|
+
usage: `${exampleUsage} get-all --slug=example-site`,
|
|
16
|
+
description: 'Retrieve a list of all local environment variables in the default table format.'
|
|
17
17
|
}, {
|
|
18
|
-
usage: `${exampleUsage} list`,
|
|
19
|
-
description: 'List the names of all environment variables.'
|
|
18
|
+
usage: `${exampleUsage} list --slug=example-site`,
|
|
19
|
+
description: 'List the names of all local environment variables.'
|
|
20
20
|
}, {
|
|
21
|
-
usage: `${exampleUsage} delete MY_VARIABLE`,
|
|
22
|
-
description: 'Delete the environment variable "MY_VARIABLE" from the environment.'
|
|
21
|
+
usage: `${exampleUsage} delete --slug=example-site MY_VARIABLE`,
|
|
22
|
+
description: 'Delete the local environment variable "MY_VARIABLE" from the environment.'
|
|
23
23
|
}];
|
|
24
24
|
(0, _command.default)({
|
|
25
25
|
requiredArgs: 0,
|
|
26
26
|
usage
|
|
27
|
-
}).command('delete', 'Delete
|
|
27
|
+
}).command('delete', 'Delete a local environment variable.').command('get', 'Retrieve the value of a local environment variable.').command('get-all', 'Retrieve the names and values of all local environment variables.').command('list', 'List the names of all local environment variables.').command('set', 'Add or update a local environment variable that begins with an uppercase letter and only includes the allowed characters A-Z, 0-9, or _.').examples(examples).argv(process.argv);
|
|
@@ -7,8 +7,10 @@ var _graphqlTag = _interopRequireDefault(require("graphql-tag"));
|
|
|
7
7
|
var _api = _interopRequireDefault(require("../lib/api"));
|
|
8
8
|
var _command = _interopRequireDefault(require("../lib/cli/command"));
|
|
9
9
|
var _format = require("../lib/cli/format");
|
|
10
|
+
var _clientFileUploader = require("../lib/client-file-uploader");
|
|
10
11
|
var _progress = require("../lib/media-import/progress");
|
|
11
12
|
var _status = require("../lib/media-import/status");
|
|
13
|
+
var _utils = require("../lib/media-import/utils");
|
|
12
14
|
var _tracker = require("../lib/tracker");
|
|
13
15
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
14
16
|
// eslint-disable-next-line no-duplicate-imports
|
|
@@ -43,10 +45,18 @@ const START_IMPORT_MUTATION = (0, _graphqlTag.default)`
|
|
|
43
45
|
const usage = 'vip import media';
|
|
44
46
|
const debug = (0, _debug.default)('vip:vip-import-media');
|
|
45
47
|
|
|
48
|
+
// Support legacy direct invocation: when users run the built binary directly
|
|
49
|
+
// like `node ./dist/bin/vip-import-media.js @1234.production /path/to/archive.tar.gz --flag`
|
|
50
|
+
// inject the `import media` subcommand into process.argv so the command parser
|
|
51
|
+
// sees the same shape as the full CLI: `vip import media ...`.
|
|
52
|
+
|
|
46
53
|
// Command examples for the `vip import media` help prompt
|
|
47
54
|
const examples = [{
|
|
48
55
|
usage: 'vip @example-app.production import media https://example.com/uploads.tar.gz',
|
|
49
56
|
description: 'Import the archived file "uploads.tar.gz" from a publicly accessible URL to a production environment.'
|
|
57
|
+
}, {
|
|
58
|
+
usage: 'vip @example-app.production import media /path/to/uploads.tar.gz',
|
|
59
|
+
description: 'Import a local archive file (e.g. .tar.gz, .tgz, .zip) from your machine into a production environment. The file will be uploaded temporarily and then imported.'
|
|
50
60
|
},
|
|
51
61
|
// Format error logs
|
|
52
62
|
{
|
|
@@ -95,13 +105,58 @@ Are you sure you want to import the contents of the URL?
|
|
|
95
105
|
overwriteExistingFiles,
|
|
96
106
|
importIntermediateImages
|
|
97
107
|
} = opts;
|
|
98
|
-
const [
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
108
|
+
const [fileNameOrURL] = args;
|
|
109
|
+
let url = '';
|
|
110
|
+
let sourceIsLocal = false;
|
|
111
|
+
if (String(fileNameOrURL).startsWith('http://') || String(fileNameOrURL).startsWith('https://')) {
|
|
112
|
+
url = fileNameOrURL;
|
|
113
|
+
// validate supported URL
|
|
114
|
+
if (!isSupportedUrl(url)) {
|
|
115
|
+
console.log(_chalk.default.red(`
|
|
116
|
+
Error:
|
|
117
|
+
Invalid URL provided: ${url}
|
|
118
|
+
Please make sure that it is a publicly accessible web URL containing an archive of the media files to import.`));
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
} else {
|
|
122
|
+
// treat as local archive path
|
|
123
|
+
if (!(await (0, _utils.isLocalArchive)(fileNameOrURL))) {
|
|
124
|
+
console.log(_chalk.default.red(`
|
|
125
|
+
Error:
|
|
126
|
+
Invalid local archive provided: ${fileNameOrURL}
|
|
127
|
+
Please make sure the file exists and is one of: .tar.gz, .tgz, .zip`));
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// upload archive to S3 and get presigned URL for import
|
|
132
|
+
sourceIsLocal = true;
|
|
133
|
+
const fileMeta = await (0, _clientFileUploader.getFileMeta)(fileNameOrURL);
|
|
134
|
+
fileMeta.fileName = fileNameOrURL;
|
|
135
|
+
const {
|
|
136
|
+
fileMeta: {
|
|
137
|
+
basename
|
|
138
|
+
},
|
|
139
|
+
checksum: uploadedMD5,
|
|
140
|
+
result
|
|
141
|
+
} = await (0, _clientFileUploader.uploadImportFileToS3)({
|
|
142
|
+
app,
|
|
143
|
+
env,
|
|
144
|
+
fileMeta,
|
|
145
|
+
progressCallback: percentage => console.log(`Upload progress: ${percentage}%`)
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
// small debug info to keep variables used
|
|
149
|
+
debug('Uploaded file basename:', basename);
|
|
150
|
+
debug('Uploaded checksum:', uploadedMD5);
|
|
151
|
+
debug('Upload result:', result && typeof result === 'object' ? Object.keys(result) : result);
|
|
152
|
+
({
|
|
153
|
+
url
|
|
154
|
+
} = await (0, _clientFileUploader.getSignedUploadRequestData)({
|
|
155
|
+
appId: app.id,
|
|
156
|
+
envId: env.id,
|
|
157
|
+
basename,
|
|
158
|
+
action: 'GetObject'
|
|
159
|
+
}));
|
|
105
160
|
}
|
|
106
161
|
const track = _tracker.trackEventWithEnv.bind(null, app.id, env.id);
|
|
107
162
|
const api = (0, _api.default)();
|
|
@@ -114,7 +169,11 @@ Error:
|
|
|
114
169
|
Importing Media into your App...
|
|
115
170
|
`;
|
|
116
171
|
console.log();
|
|
117
|
-
|
|
172
|
+
if (sourceIsLocal) {
|
|
173
|
+
console.log(`Importing local archive: ${fileNameOrURL} (uploaded to temporary URL)`);
|
|
174
|
+
} else {
|
|
175
|
+
console.log(`Importing archive from: ${url}`);
|
|
176
|
+
}
|
|
118
177
|
console.log(`to: ${env.primaryDomain.name} (${(0, _format.formatEnvironment)(env.type)})`);
|
|
119
178
|
try {
|
|
120
179
|
await api.mutate({
|
|
@@ -591,7 +591,7 @@ Processing the SQL import for your environment...
|
|
|
591
591
|
},
|
|
592
592
|
checksum: uploadedMD5,
|
|
593
593
|
result
|
|
594
|
-
} = await (0, _clientFileUploader.
|
|
594
|
+
} = await (0, _clientFileUploader.uploadImportFileToS3)({
|
|
595
595
|
app,
|
|
596
596
|
env,
|
|
597
597
|
fileMeta,
|
|
@@ -8,9 +8,11 @@ var _vipSearchReplace = require("@automattic/vip-search-replace");
|
|
|
8
8
|
var _chalk = _interopRequireDefault(require("chalk"));
|
|
9
9
|
var _debug = _interopRequireDefault(require("debug"));
|
|
10
10
|
var _fs = _interopRequireDefault(require("fs"));
|
|
11
|
+
var _graphqlTag = _interopRequireDefault(require("graphql-tag"));
|
|
11
12
|
var _promises = require("node:stream/promises");
|
|
12
13
|
var _devEnvImportSql = require("./dev-env-import-sql");
|
|
13
14
|
var _exportSql = require("./export-sql");
|
|
15
|
+
var _api = _interopRequireDefault(require("../lib/api"));
|
|
14
16
|
var _backupStorageAvailability = require("../lib/backup-storage-availability/backup-storage-availability");
|
|
15
17
|
var exit = _interopRequireWildcard(require("../lib/cli/exit"));
|
|
16
18
|
var _clientFileUploader = require("../lib/client-file-uploader");
|
|
@@ -83,6 +85,24 @@ async function extractSiteUrls(sqlFile) {
|
|
|
83
85
|
readInterface.on('error', reject);
|
|
84
86
|
});
|
|
85
87
|
}
|
|
88
|
+
const SITE_URLS_QUERY = (0, _graphqlTag.default)`
|
|
89
|
+
query SiteUrlsQuery($appId: Int!, $environmentId: Int!, $after: String, $first: Int!) {
|
|
90
|
+
app(id: $appId) {
|
|
91
|
+
environments(id: $environmentId) {
|
|
92
|
+
wpSitesSDS(after: $after, first: $first) {
|
|
93
|
+
total
|
|
94
|
+
nextCursor
|
|
95
|
+
nodes {
|
|
96
|
+
id
|
|
97
|
+
blogId
|
|
98
|
+
homeUrl
|
|
99
|
+
siteUrl
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
`;
|
|
86
106
|
class DevEnvSyncSQLCommand {
|
|
87
107
|
tmpDir;
|
|
88
108
|
siteUrls = [];
|
|
@@ -90,6 +110,7 @@ class DevEnvSyncSQLCommand {
|
|
|
90
110
|
liveBackupCopyCLIOptions;
|
|
91
111
|
_track;
|
|
92
112
|
_sqlDumpType;
|
|
113
|
+
sdsSiteUrls = [];
|
|
93
114
|
|
|
94
115
|
/**
|
|
95
116
|
* Creates a new instance of the command
|
|
@@ -167,13 +188,51 @@ class DevEnvSyncSQLCommand {
|
|
|
167
188
|
await (0, _promises.pipeline)(streams);
|
|
168
189
|
_fs.default.renameSync(outputFile, this.sqlFile);
|
|
169
190
|
}
|
|
191
|
+
async getSiteUrlsFromSDS() {
|
|
192
|
+
if (this.env.isMultisite && (!this.env.wpSitesSDS?.nodes || !this.env.wpSitesSDS?.total || this.env.wpSitesSDS.nodes.length < this.env.wpSitesSDS.total)) {
|
|
193
|
+
return this.fetchAllSites(Number(this.app.id), Number(this.env.id));
|
|
194
|
+
}
|
|
195
|
+
return this.env.wpSitesSDS?.nodes?.filter(node => Boolean(node)) ?? [];
|
|
196
|
+
}
|
|
197
|
+
fetchSitesPage(api, appId, environmentId, after) {
|
|
198
|
+
return api.query({
|
|
199
|
+
query: SITE_URLS_QUERY,
|
|
200
|
+
variables: {
|
|
201
|
+
first: 100,
|
|
202
|
+
after,
|
|
203
|
+
appId,
|
|
204
|
+
environmentId
|
|
205
|
+
}
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
async fetchAllSites(appId, environmentId) {
|
|
209
|
+
const api = (0, _api.default)();
|
|
210
|
+
let after = null;
|
|
211
|
+
const allSites = [];
|
|
212
|
+
let total;
|
|
213
|
+
console.log('Fetching list of sites for database sync...');
|
|
214
|
+
do {
|
|
215
|
+
// eslint-disable-next-line no-await-in-loop
|
|
216
|
+
const res = await this.fetchSitesPage(api, appId, environmentId, after);
|
|
217
|
+
if (res.data.app?.environments?.[0]?.wpSitesSDS?.nodes) {
|
|
218
|
+
const wpSitesSDS = res.data.app.environments[0].wpSitesSDS;
|
|
219
|
+
allSites.push(...res.data.app.environments[0].wpSitesSDS.nodes.filter(node => Boolean(node)));
|
|
220
|
+
after = wpSitesSDS.nextCursor;
|
|
221
|
+
total = Number(wpSitesSDS.total);
|
|
222
|
+
console.log(`Fetched ${allSites.length} of ${total} sites...`);
|
|
223
|
+
} else {
|
|
224
|
+
after = null;
|
|
225
|
+
}
|
|
226
|
+
} while (after);
|
|
227
|
+
return allSites;
|
|
228
|
+
}
|
|
170
229
|
generateSearchReplaceMap() {
|
|
171
230
|
this.searchReplaceMap = {};
|
|
172
231
|
for (const url of this.siteUrls) {
|
|
173
232
|
this.searchReplaceMap[stripProtocol(url)] = stripProtocol(replaceDomain(url, this.landoDomain));
|
|
174
233
|
}
|
|
175
|
-
const networkSites = this.
|
|
176
|
-
if (!networkSites) return;
|
|
234
|
+
const networkSites = this.sdsSiteUrls;
|
|
235
|
+
if (!networkSites.length) return;
|
|
177
236
|
const primaryUrl = networkSites.find(site => site?.blogId === 1)?.homeUrl;
|
|
178
237
|
const primaryDomain = primaryUrl ? new URL(primaryUrl).hostname : '';
|
|
179
238
|
debug('Network sites: %j, primary URL: %s, primary domain: %s', networkSites.map(site => ({
|
|
@@ -218,8 +277,8 @@ class DevEnvSyncSQLCommand {
|
|
|
218
277
|
await importCommand.run();
|
|
219
278
|
}
|
|
220
279
|
fixBlogsTableQuery() {
|
|
221
|
-
const networkSites = this.
|
|
222
|
-
if (!networkSites) {
|
|
280
|
+
const networkSites = this.sdsSiteUrls;
|
|
281
|
+
if (!networkSites.length) {
|
|
223
282
|
return '';
|
|
224
283
|
}
|
|
225
284
|
const prologue = `
|
|
@@ -301,6 +360,17 @@ DROP PROCEDURE vip_sync_update_blog_domains;
|
|
|
301
360
|
});
|
|
302
361
|
exit.withError(`Error extracting site URLs: ${error.message}`);
|
|
303
362
|
}
|
|
363
|
+
try {
|
|
364
|
+
this.sdsSiteUrls = await this.getSiteUrlsFromSDS();
|
|
365
|
+
} catch (err) {
|
|
366
|
+
const error = err;
|
|
367
|
+
await this.track('error', {
|
|
368
|
+
error_type: 'get_site_urls_from_sds',
|
|
369
|
+
error_message: error.message,
|
|
370
|
+
stack: error.stack
|
|
371
|
+
});
|
|
372
|
+
exit.withError(`Error getting site URLs from SDS: ${error.message}`);
|
|
373
|
+
}
|
|
304
374
|
console.log('Generating search-replace configuration...');
|
|
305
375
|
this.generateSearchReplaceMap();
|
|
306
376
|
try {
|
|
@@ -335,7 +335,7 @@ class ExportSQLCommand {
|
|
|
335
335
|
if (this.liveBackupCopyCLIOptions?.useLiveBackupCopy) {
|
|
336
336
|
const result = await this.generateLiveBackupCopy();
|
|
337
337
|
url = result.url;
|
|
338
|
-
size = result.size;
|
|
338
|
+
size = Number(result.size);
|
|
339
339
|
} else {
|
|
340
340
|
url = await this.runBackup();
|
|
341
341
|
const exportJob = await this.getExportJob();
|
|
@@ -349,7 +349,7 @@ class ExportSQLCommand {
|
|
|
349
349
|
size = Number(bytesWrittenMeta.value);
|
|
350
350
|
}
|
|
351
351
|
const storageConfirmed = await this.progressTracker.handleContinuePrompt(async setPromptShown => {
|
|
352
|
-
const status = await this.confirmEnoughStorage(size);
|
|
352
|
+
const status = await this.confirmEnoughStorage(Number(size));
|
|
353
353
|
if (status.isPromptShown) {
|
|
354
354
|
setPromptShown();
|
|
355
355
|
}
|
package/dist/lib/api.js
CHANGED
|
@@ -5,11 +5,15 @@ exports.PRODUCTION_API_HOST = exports.API_URL = exports.API_HOST = void 0;
|
|
|
5
5
|
exports.default = API;
|
|
6
6
|
exports.disableGlobalGraphQLErrorHandling = disableGlobalGraphQLErrorHandling;
|
|
7
7
|
exports.enableGlobalGraphQLErrorHandling = enableGlobalGraphQLErrorHandling;
|
|
8
|
+
exports.shouldRetryRequest = shouldRetryRequest;
|
|
8
9
|
var _core = require("@apollo/client/core");
|
|
9
10
|
var _context = require("@apollo/client/link/context");
|
|
10
11
|
var _core2 = require("@apollo/client/link/core");
|
|
11
12
|
var _error = require("@apollo/client/link/error");
|
|
13
|
+
var _retry = require("@apollo/client/link/retry");
|
|
12
14
|
var _chalk = _interopRequireDefault(require("chalk"));
|
|
15
|
+
var _debug = _interopRequireDefault(require("debug"));
|
|
16
|
+
var _nodeFetch = require("node-fetch");
|
|
13
17
|
var _http = _interopRequireDefault(require("./api/http"));
|
|
14
18
|
var _env = _interopRequireDefault(require("./env"));
|
|
15
19
|
var _token = _interopRequireDefault(require("./token"));
|
|
@@ -21,12 +25,39 @@ const PRODUCTION_API_HOST = exports.PRODUCTION_API_HOST = 'https://api.wpvip.com
|
|
|
21
25
|
const API_HOST = exports.API_HOST = process.env.API_HOST || PRODUCTION_API_HOST; // NOSONAR
|
|
22
26
|
const API_URL = exports.API_URL = `${API_HOST}/graphql`;
|
|
23
27
|
let globalGraphQLErrorHandlingEnabled = true;
|
|
28
|
+
const RETRY_LINK_MAX_ATTEMPTS = 5;
|
|
29
|
+
const RETRY_LINK_INITIAL_DELAY_MS = 1000; // 1 second
|
|
30
|
+
const RETRY_LINK_MAX_DELAY_MS = 5000; // 5 seconds
|
|
31
|
+
|
|
32
|
+
const debug = (0, _debug.default)('@automattic/vip:http:graphql');
|
|
24
33
|
function disableGlobalGraphQLErrorHandling() {
|
|
25
34
|
globalGraphQLErrorHandlingEnabled = false;
|
|
26
35
|
}
|
|
27
36
|
function enableGlobalGraphQLErrorHandling() {
|
|
28
37
|
globalGraphQLErrorHandlingEnabled = true;
|
|
29
38
|
}
|
|
39
|
+
function shouldRetryRequest(attempt, operation, error) {
|
|
40
|
+
const debugSuffix = `Operation: ${operation.operationName}. Attempt: ${attempt}.`;
|
|
41
|
+
if (!error || operation.query.definitions.some(def => def.kind === 'OperationDefinition' && def.operation !== 'query')) {
|
|
42
|
+
debug(`Request failed. ${debugSuffix}`);
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
if (attempt > RETRY_LINK_MAX_ATTEMPTS) {
|
|
46
|
+
debug(`Request failed and max retry attempts reached. ${debugSuffix}`, error);
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
if (error instanceof _nodeFetch.FetchError && error.code === 'ECONNREFUSED') {
|
|
50
|
+
debug(`Request failed. Retrying request due to connection refused error. ${debugSuffix}`);
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
const statusCode = error?.statusCode;
|
|
54
|
+
if (statusCode && statusCode !== 429 && statusCode < 500) {
|
|
55
|
+
debug(`Request failed. Status code: ${statusCode}. ${debugSuffix}`, error);
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
debug(`Request failed. Retrying request due to server error. ${debugSuffix}`, error);
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
30
61
|
function isServerError(networkError) {
|
|
31
62
|
if (!networkError) {
|
|
32
63
|
return false;
|
|
@@ -35,7 +66,8 @@ function isServerError(networkError) {
|
|
|
35
66
|
}
|
|
36
67
|
function API({
|
|
37
68
|
exitOnError = true,
|
|
38
|
-
silenceAuthErrors = false
|
|
69
|
+
silenceAuthErrors = false,
|
|
70
|
+
customRetryLink
|
|
39
71
|
} = {}) {
|
|
40
72
|
const errorLink = (0, _error.onError)(({
|
|
41
73
|
networkError,
|
|
@@ -90,8 +122,15 @@ function API({
|
|
|
90
122
|
agent: proxyAgent
|
|
91
123
|
}
|
|
92
124
|
});
|
|
125
|
+
const retryLink = new _retry.RetryLink({
|
|
126
|
+
delay: {
|
|
127
|
+
initial: RETRY_LINK_INITIAL_DELAY_MS,
|
|
128
|
+
max: RETRY_LINK_MAX_DELAY_MS
|
|
129
|
+
},
|
|
130
|
+
attempts: shouldRetryRequest
|
|
131
|
+
});
|
|
93
132
|
return new _core.ApolloClient({
|
|
94
|
-
link: _core2.ApolloLink.from([withToken, errorLink, authLink, httpLink]),
|
|
133
|
+
link: _core2.ApolloLink.from([withToken, errorLink, customRetryLink ? customRetryLink : retryLink, authLink, httpLink]),
|
|
95
134
|
cache: new _core.InMemoryCache({
|
|
96
135
|
typePolicies: {
|
|
97
136
|
WPSite: {
|
|
@@ -8,9 +8,10 @@ exports.getFileHash = void 0;
|
|
|
8
8
|
exports.getFileMeta = getFileMeta;
|
|
9
9
|
exports.getFileSize = getFileSize;
|
|
10
10
|
exports.getPartBoundaries = getPartBoundaries;
|
|
11
|
+
exports.getSignedUploadRequestData = getSignedUploadRequestData;
|
|
11
12
|
exports.isFile = isFile;
|
|
12
13
|
exports.unzipFile = void 0;
|
|
13
|
-
exports.
|
|
14
|
+
exports.uploadImportFileToS3 = uploadImportFileToS3;
|
|
14
15
|
exports.uploadParts = uploadParts;
|
|
15
16
|
var _chalk = _interopRequireDefault(require("chalk"));
|
|
16
17
|
var _crypto = require("crypto");
|
|
@@ -121,7 +122,7 @@ async function getFileMeta(fileName) {
|
|
|
121
122
|
isCompressed
|
|
122
123
|
};
|
|
123
124
|
}
|
|
124
|
-
async function
|
|
125
|
+
async function uploadImportFileToS3({
|
|
125
126
|
app,
|
|
126
127
|
env,
|
|
127
128
|
fileMeta,
|
|
@@ -28,7 +28,7 @@ const DEV_ENVIRONMENT_PHP_VERSIONS = exports.DEV_ENVIRONMENT_PHP_VERSIONS = {
|
|
|
28
28
|
},
|
|
29
29
|
8.4: {
|
|
30
30
|
image: 'ghcr.io/automattic/vip-container-images/php-fpm:8.4',
|
|
31
|
-
label: '8.4
|
|
31
|
+
label: '8.4'
|
|
32
32
|
}
|
|
33
33
|
};
|
|
34
34
|
const DEV_ENVIRONMENT_DEFAULTS = exports.DEV_ENVIRONMENT_DEFAULTS = {
|
|
@@ -36,4 +36,4 @@ const DEV_ENVIRONMENT_DEFAULTS = exports.DEV_ENVIRONMENT_DEFAULTS = {
|
|
|
36
36
|
multisite: false,
|
|
37
37
|
phpVersion: Object.keys(DEV_ENVIRONMENT_PHP_VERSIONS)[0]
|
|
38
38
|
};
|
|
39
|
-
const DEV_ENVIRONMENT_VERSION = exports.DEV_ENVIRONMENT_VERSION = '2.3.
|
|
39
|
+
const DEV_ENVIRONMENT_VERSION = exports.DEV_ENVIRONMENT_VERSION = '2.3.1';
|
|
@@ -137,7 +137,7 @@ async function getDownloadURL({
|
|
|
137
137
|
}
|
|
138
138
|
return {
|
|
139
139
|
url: result.data.generateLiveBackupCopyDownloadURL.url,
|
|
140
|
-
size: result.data.generateLiveBackupCopyDownloadURL.size
|
|
140
|
+
size: Number(result.data.generateLiveBackupCopyDownloadURL.size)
|
|
141
141
|
};
|
|
142
142
|
} catch (error) {
|
|
143
143
|
if (error instanceof _utils.PollingTimeoutError) {
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
exports.__esModule = true;
|
|
4
|
+
exports.isLocalArchive = isLocalArchive;
|
|
5
|
+
var _promises = require("node:fs/promises");
|
|
6
|
+
async function isLocalArchive(filePath) {
|
|
7
|
+
const lower = filePath.toLowerCase();
|
|
8
|
+
const isArchive = lower.endsWith('.tar.gz') || lower.endsWith('.tgz') || lower.endsWith('.zip');
|
|
9
|
+
if (!isArchive) {
|
|
10
|
+
return false;
|
|
11
|
+
}
|
|
12
|
+
try {
|
|
13
|
+
const fileStat = await (0, _promises.stat)(filePath);
|
|
14
|
+
return fileStat.isFile();
|
|
15
|
+
} catch {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
}
|
package/dist/lib/utils.js
CHANGED
|
@@ -6,6 +6,7 @@ exports.getAbsolutePath = getAbsolutePath;
|
|
|
6
6
|
exports.makeTempDir = makeTempDir;
|
|
7
7
|
exports.parseApiError = parseApiError;
|
|
8
8
|
exports.pollUntil = pollUntil;
|
|
9
|
+
exports.splitKeyValueString = splitKeyValueString;
|
|
9
10
|
var _debug = _interopRequireDefault(require("debug"));
|
|
10
11
|
var _fs = _interopRequireDefault(require("fs"));
|
|
11
12
|
var _promises = require("node:timers/promises");
|
|
@@ -95,4 +96,13 @@ function parseApiError(err) {
|
|
|
95
96
|
return err?.message;
|
|
96
97
|
}
|
|
97
98
|
return null;
|
|
99
|
+
}
|
|
100
|
+
function splitKeyValueString(str, delimiter = '=') {
|
|
101
|
+
const delimiterIndex = str.indexOf(delimiter);
|
|
102
|
+
if (delimiterIndex === -1) {
|
|
103
|
+
return [str.trim(), ''];
|
|
104
|
+
}
|
|
105
|
+
const key = str.slice(0, delimiterIndex).trim();
|
|
106
|
+
const value = str.slice(delimiterIndex + delimiter.length).trim();
|
|
107
|
+
return [key, value];
|
|
98
108
|
}
|