easy-devops 1.3.5 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/ui/assets/index-Bu7af-Kj.css +1 -0
- package/dist/ui/assets/index-gndgYe3K.js +532 -0
- package/dist/ui/assets/socket-Bv0wCEgi.js +65 -0
- package/dist/ui/index.html +2 -2
- package/package.json +2 -2
- package/src/cli/index.js +8 -0
- package/src/cli/managers/domain-manager.js +37 -6
- package/src/cli/managers/nginx-manager.js +14 -5
- package/src/cli/managers/node-manager.js +19 -13
- package/src/cli/managers/ssl-manager.js +89 -26
- package/src/cli/menu.js +8 -2
- package/src/cli/menus/cleanup.js +4 -2
- package/src/cli/menus/dashboard.js +22 -15
- package/src/cli/menus/hub.js +381 -0
- package/src/cli/menus/runtime.js +12 -0
- package/src/cli/menus/service.js +28 -10
- package/src/cli/menus/settings.js +46 -12
- package/src/cli/menus/uninstall.js +13 -5
- package/src/cli/menus/update.js +24 -14
- package/src/cli/ui/page.js +123 -0
- package/src/core/cert-operations.js +34 -4
- package/src/core/cert-reader.js +67 -22
- package/src/core/cert-renewal.js +65 -9
- package/src/core/cert-suggest.js +120 -0
- package/src/core/cleanup.js +221 -1
- package/src/core/cloudflare-ips.js +161 -0
- package/src/core/daemon/ipc.js +1 -1
- package/src/core/daemon/lock.js +296 -9
- package/src/core/dashboardProcess.js +134 -6
- package/src/core/deploy/apps.js +36 -62
- package/src/core/deploy/deployKey.js +331 -0
- package/src/core/deploy/domainLink.js +280 -23
- package/src/core/deploy/engine.js +382 -63
- package/src/core/deploy/ezfile.js +496 -29
- package/src/core/deploy/githubApi.js +100 -13
- package/src/core/deploy/hubCommands.js +158 -18
- package/src/core/deploy/status.js +28 -1
- package/src/core/domains-db.js +1 -0
- package/src/core/hub/agent.js +238 -32
- package/src/core/hub/deployProgress.js +411 -0
- package/src/core/hubConnect.js +205 -0
- package/src/core/hubPermissions.js +48 -10
- package/src/core/jobs/bus.js +1 -1
- package/src/core/jobs/owner.js +268 -6
- package/src/core/nginx-conf-generator.js +124 -19
- package/src/core/notifications/claim.js +26 -3
- package/src/core/permissions.js +277 -59
- package/src/core/platform.js +48 -2
- package/src/core/runner.js +96 -40
- package/src/core/ssl/preflight.js +548 -0
- package/src/core/ssl/progress.js +196 -0
- package/src/core/ssl-issuer.js +502 -148
- package/src/core/validators.js +24 -0
- package/src/dashboard/controllers/authController.js +12 -5
- package/src/dashboard/controllers/deploymentsController.js +76 -11
- package/src/dashboard/controllers/domainsController.js +35 -1
- package/src/dashboard/controllers/hubController.js +4 -3
- package/src/dashboard/controllers/settingsController.js +73 -68
- package/src/dashboard/controllers/sslController.js +135 -15
- package/src/dashboard/public/js/app.js +42 -7
- package/src/dashboard/routes/deployments.js +2 -0
- package/src/dashboard/routes/domains.js +1 -0
- package/src/dashboard/server.js +73 -8
- package/src/dashboard/socketManager.js +19 -25
- package/src/dashboard/views/partials/settings-panel.ejs +29 -10
- package/dist/ui/assets/index-BUpPxWmp.css +0 -1
- package/dist/ui/assets/index-DD0JjK0Q.js +0 -475
- package/dist/ui/assets/socket-4tfdap-e.js +0 -65
|
@@ -7,14 +7,27 @@
|
|
|
7
7
|
*
|
|
8
8
|
* 1. fetch a fresh checkout into releases/<id> — never over the live one
|
|
9
9
|
* 2. settings ez.json + package.json + the app's overrides (ezfile.js)
|
|
10
|
-
* 3. pre hook →
|
|
10
|
+
* 3. pre hook → the steps list (install/build/migrate/custom) in order
|
|
11
11
|
* 4. switch stop the old process, start the new one under PM2
|
|
12
12
|
* 5. health the app must answer (or stay up) before it counts
|
|
13
13
|
* 6. post hook, save the PM2 list, prune old releases
|
|
14
14
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
15
|
+
* Every step runs in the new release with the build's environment, before
|
|
16
|
+
* anything is started. `migrate` keeps its forward-only hint: rolling the code
|
|
17
|
+
* back does not undo a migration that already ran, which is why the hint on a
|
|
18
|
+
* failure says to fix the migration rather than expect the rollback to undo it.
|
|
19
|
+
* A disabled step is logged as skipped.
|
|
20
|
+
*
|
|
21
|
+
* Nothing before step 4 touches what is running, so a broken build or a failed
|
|
22
|
+
* migration costs nothing. If step 4 or 5 fails and there is a previous
|
|
23
|
+
* release, it is started again — the same files that were running before, not a
|
|
24
|
+
* rebuild.
|
|
25
|
+
*
|
|
26
|
+
* Step 5 watches PM2 as well as HTTP. A release that answers once and then
|
|
27
|
+
* crash-loops, or that PM2 reports errored/stopped, is a boot crash: the
|
|
28
|
+
* previous release is restored, and the run records why, the last PM2 log
|
|
29
|
+
* lines (with the app's environment values redacted) and whether that rollback
|
|
30
|
+
* succeeded. A first release has nothing to return to, so it is stopped.
|
|
18
31
|
*
|
|
19
32
|
* ── What this does not do ────────────────────────────────────────────────────
|
|
20
33
|
*
|
|
@@ -45,7 +58,8 @@ import { createDeployEvent } from '../events.js';
|
|
|
45
58
|
import { sendNotification as defaultSendNotification } from '../notifier.js';
|
|
46
59
|
import { publish } from '../jobs/bus.js';
|
|
47
60
|
|
|
48
|
-
import { getApp, deleteApp, pm2Name, AUTH_TYPES,
|
|
61
|
+
import { getApp, deleteApp, pm2Name, AUTH_TYPES, validateRef, setAppDir } from './apps.js';
|
|
62
|
+
import { sshCommand } from './deployKey.js';
|
|
49
63
|
import { deleteLinkedDomain, syncLinkedDomain } from './domainLink.js';
|
|
50
64
|
import {
|
|
51
65
|
EZ_FILE_LOCATIONS, parseEzFile, resolveSettings, parseStartCommand,
|
|
@@ -66,6 +80,20 @@ const HEALTH_POLL_MS = 2000;
|
|
|
66
80
|
/** Without a port there is nothing to ask, so the app only has to stay up this long. */
|
|
67
81
|
const NO_PORT_SETTLE_MS = 8000;
|
|
68
82
|
const ROLLBACK_HEALTH_TIMEOUT_S = 30;
|
|
83
|
+
/**
|
|
84
|
+
* How long a healthy answer must also stay online without a PM2 restart.
|
|
85
|
+
* PM2's own `min_uptime` is 5 s, so a release that dies inside that window is
|
|
86
|
+
* one PM2 itself considers unstable; matching it keeps the two judgements
|
|
87
|
+
* aligned.
|
|
88
|
+
*/
|
|
89
|
+
const BOOT_CONFIRM_MS = 5000;
|
|
90
|
+
/** PM2 out+error lines kept in the run record when a release crashes at boot. */
|
|
91
|
+
const BOOT_LOG_LINES = 40;
|
|
92
|
+
|
|
93
|
+
const MIGRATE_HINT =
|
|
94
|
+
'Migrations are forward-only: rolling the code back cannot undo a migration that already ran. ' +
|
|
95
|
+
'Fix the migration, keep it backward-compatible for one release, and deploy again. ' +
|
|
96
|
+
'The release that was running was not touched.';
|
|
69
97
|
|
|
70
98
|
export const RUN_STATUS = Object.freeze({
|
|
71
99
|
RUNNING: 'running',
|
|
@@ -76,9 +104,11 @@ export const RUN_STATUS = Object.freeze({
|
|
|
76
104
|
|
|
77
105
|
/** A failure with the step it happened in, so the run record can say where. */
|
|
78
106
|
class StepError extends Error {
|
|
79
|
-
constructor(step, message) {
|
|
107
|
+
constructor(step, message, { bootCrash = false, reason = null } = {}) {
|
|
80
108
|
super(message);
|
|
81
109
|
this.step = step;
|
|
110
|
+
this.bootCrash = bootCrash;
|
|
111
|
+
this.reason = reason;
|
|
82
112
|
}
|
|
83
113
|
}
|
|
84
114
|
|
|
@@ -118,6 +148,31 @@ export function isHealthyStatus(status, strict) {
|
|
|
118
148
|
return strict ? status >= 200 && status < 400 : status >= 100 && status < 500;
|
|
119
149
|
}
|
|
120
150
|
|
|
151
|
+
/**
|
|
152
|
+
* Removes the app's secret values from text that is about to leave the engine.
|
|
153
|
+
*
|
|
154
|
+
* PM2's own logs are the app's output, and an app that prints its configuration
|
|
155
|
+
* — or a crash dump that includes it — would otherwise put a database URL or a
|
|
156
|
+
* token into a run record the dashboard and the Hub can read. Every non-empty
|
|
157
|
+
* value of an environment variable on the app is replaced, longest first, so a
|
|
158
|
+
* secret that contains another one is not half-redacted. The auth token is
|
|
159
|
+
* included for the same reason even though it lives outside `env`.
|
|
160
|
+
*
|
|
161
|
+
* @param {string} text
|
|
162
|
+
* @param {Object<string, *>} [env]
|
|
163
|
+
* @param {string|null} [token]
|
|
164
|
+
* @returns {string}
|
|
165
|
+
*/
|
|
166
|
+
export function redactSecrets(text, env = {}, token = null) {
|
|
167
|
+
let out = String(text ?? '');
|
|
168
|
+
const values = [...Object.values(env ?? {}), token]
|
|
169
|
+
.filter((v) => typeof v === 'string' && v.length > 0)
|
|
170
|
+
.sort((a, b) => b.length - a.length);
|
|
171
|
+
|
|
172
|
+
for (const value of values) out = out.split(value).join('***');
|
|
173
|
+
return out;
|
|
174
|
+
}
|
|
175
|
+
|
|
121
176
|
// ─── The engine ───────────────────────────────────────────────────────────────
|
|
122
177
|
|
|
123
178
|
function resolveDeps(deps = {}) {
|
|
@@ -192,13 +247,17 @@ function gitAuth(app) {
|
|
|
192
247
|
}
|
|
193
248
|
|
|
194
249
|
if (app.auth?.type === AUTH_TYPES.DEPLOY_KEY) {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
250
|
+
// github.com is checked against GitHub's pinned host keys; see deployKey.js.
|
|
251
|
+
let sshCmd;
|
|
252
|
+
try {
|
|
253
|
+
sshCmd = sshCommand(app.id, app.repoUrl);
|
|
254
|
+
} catch (err) {
|
|
255
|
+
throw new StepError('fetch', err.message);
|
|
256
|
+
}
|
|
257
|
+
if (!sshCmd) {
|
|
258
|
+
throw new StepError('fetch', 'This app uses a deploy key, but none has been generated yet. Generate it in the app settings, add it to the repository, then deploy again.');
|
|
259
|
+
}
|
|
260
|
+
env.GIT_SSH_COMMAND = sshCmd;
|
|
202
261
|
}
|
|
203
262
|
|
|
204
263
|
return { env, flags: '', display: '' };
|
|
@@ -238,7 +297,12 @@ export function loadProjectSettings(dir, overrides = {}) {
|
|
|
238
297
|
};
|
|
239
298
|
|
|
240
299
|
const resolved = resolveSettings({ ez, pkg, files, overrides });
|
|
241
|
-
return {
|
|
300
|
+
return {
|
|
301
|
+
...resolved,
|
|
302
|
+
// ez.json warnings plus the runtime warnings (cluster mode's stateless note).
|
|
303
|
+
warnings: [...warnings, ...(resolved.warnings ?? [])],
|
|
304
|
+
ezLocation,
|
|
305
|
+
};
|
|
242
306
|
}
|
|
243
307
|
|
|
244
308
|
// ─── Persistent paths ────────────────────────────────────────────────────────
|
|
@@ -602,6 +666,13 @@ function writePm2Config(dir, app, settings, env) {
|
|
|
602
666
|
entry.env[LAUNCHER_COMMAND_ENV] = start.command;
|
|
603
667
|
}
|
|
604
668
|
|
|
669
|
+
// Fork (the default) is left exactly as it was: PM2's own default. Cluster
|
|
670
|
+
// needs both fields, and a rollback reads them back from the release meta.
|
|
671
|
+
if (settings.execMode === 'cluster') {
|
|
672
|
+
entry.exec_mode = 'cluster';
|
|
673
|
+
entry.instances = settings.instances === 'max' ? 'max' : (settings.instances ?? 1);
|
|
674
|
+
}
|
|
675
|
+
|
|
605
676
|
const file = path.join(dir, PM2_CONFIG_FILE);
|
|
606
677
|
// Holds the app's environment, so owner-only like the deploy key.
|
|
607
678
|
fs.writeFileSync(file, JSON.stringify({ apps: [entry] }, null, 2), { mode: 0o600 });
|
|
@@ -620,9 +691,14 @@ function releaseHealth(dir) {
|
|
|
620
691
|
sha: typeof meta.sha === 'string' ? meta.sha : null,
|
|
621
692
|
ref: typeof meta.ref === 'string' ? meta.ref : null,
|
|
622
693
|
persist: Array.isArray(meta.persist) ? meta.persist.filter((p) => typeof p === 'string') : [],
|
|
694
|
+
execMode: meta.execMode === 'cluster' ? 'cluster' : 'fork',
|
|
695
|
+
instances: meta.instances === 'max' || Number.isInteger(meta.instances) ? meta.instances : 1,
|
|
623
696
|
};
|
|
624
697
|
} catch {
|
|
625
|
-
return {
|
|
698
|
+
return {
|
|
699
|
+
port: null, healthPath: '/', strict: false, sha: null, ref: null, persist: [],
|
|
700
|
+
execMode: 'fork', instances: 1,
|
|
701
|
+
};
|
|
626
702
|
}
|
|
627
703
|
}
|
|
628
704
|
|
|
@@ -635,70 +711,230 @@ export function describeRelease(appId, releaseId) {
|
|
|
635
711
|
id: releaseId,
|
|
636
712
|
sha: meta.sha,
|
|
637
713
|
ref: meta.ref,
|
|
714
|
+
execMode: meta.execMode,
|
|
715
|
+
instances: meta.instances,
|
|
638
716
|
startable: fs.existsSync(path.join(dir, PM2_CONFIG_FILE)),
|
|
639
717
|
};
|
|
640
718
|
}
|
|
641
719
|
|
|
720
|
+
/**
|
|
721
|
+
* Reads one app's PM2 status through core/runner.js, as a list of instances.
|
|
722
|
+
*
|
|
723
|
+
* A cluster app has one PM2 entry per worker, so statusAll is preferred. The
|
|
724
|
+
* older `status` and `list` fallbacks keep injected runners working; they
|
|
725
|
+
* report one entry, which is all a fork app has anyway.
|
|
726
|
+
*
|
|
727
|
+
* @returns {Promise<{ read: boolean, apps: Object[] }>}
|
|
728
|
+
*/
|
|
729
|
+
async function readPm2Status(name, pm2Path, d) {
|
|
730
|
+
if (typeof d.runner.statusAll === 'function') {
|
|
731
|
+
const r = await d.runner.statusAll(name, { run: d.run, pm2Path });
|
|
732
|
+
if (!r?.ok) return { read: false, apps: [] };
|
|
733
|
+
return { read: true, apps: Array.isArray(r.apps) ? r.apps : [] };
|
|
734
|
+
}
|
|
735
|
+
if (typeof d.runner.status === 'function') {
|
|
736
|
+
const r = await d.runner.status(name, { run: d.run, pm2Path });
|
|
737
|
+
if (!r?.ok) return { read: false, apps: [] };
|
|
738
|
+
return { read: true, apps: r.app ? [r.app] : [] };
|
|
739
|
+
}
|
|
740
|
+
const listed = await d.runner.list({ run: d.run, pm2Path });
|
|
741
|
+
if (!listed.ok) return { read: false, apps: [] };
|
|
742
|
+
return { read: true, apps: listed.apps.filter((a) => a.name === name) };
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/**
|
|
746
|
+
* Turns one PM2 observation into a boot-crash reason, or null when it is fine.
|
|
747
|
+
*
|
|
748
|
+
* `baseline` maps each instance to the restart count on the first observation
|
|
749
|
+
* after the start. Anything above its baseline means that worker exited at
|
|
750
|
+
* least once; PM2's state `errored`/`stopped` means it has given up. Both are
|
|
751
|
+
* boot crashes even when an HTTP probe happened to answer once, which is the
|
|
752
|
+
* case the health check alone cannot see. A cluster app is only healthy when
|
|
753
|
+
* EVERY worker is.
|
|
754
|
+
*
|
|
755
|
+
* @param {{ read: boolean, apps: Object[] }} observed
|
|
756
|
+
* @param {Map<number|string, number>|null} baseline
|
|
757
|
+
* @param {{ port?: number|null }} ctx
|
|
758
|
+
* @returns {string|null}
|
|
759
|
+
*/
|
|
760
|
+
function bootCrashReason(observed, baseline, { port = null } = {}) {
|
|
761
|
+
if (!observed.read) return null;
|
|
762
|
+
const apps = Array.isArray(observed.apps) ? observed.apps : [];
|
|
763
|
+
if (apps.length === 0) return 'crashed at boot: PM2 no longer lists the process';
|
|
764
|
+
|
|
765
|
+
for (let i = 0; i < apps.length; i++) {
|
|
766
|
+
const app = apps[i];
|
|
767
|
+
const key = app.pmId ?? i;
|
|
768
|
+
const restarts = Number.isFinite(app.restarts) ? app.restarts : 0;
|
|
769
|
+
|
|
770
|
+
if (baseline && baseline.has(key) && restarts > baseline.get(key)) {
|
|
771
|
+
return `crashed at boot: restarting ${restarts} times`;
|
|
772
|
+
}
|
|
773
|
+
if (app.status === 'errored' || app.status === 'stopped') {
|
|
774
|
+
const where = port ? ` instead of listening on port ${port}` : '';
|
|
775
|
+
return `crashed at boot: process is "${app.status}" — it exited${where}`;
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
return null;
|
|
780
|
+
}
|
|
781
|
+
|
|
642
782
|
/**
|
|
643
783
|
* Waits for a freshly started app to be healthy.
|
|
644
|
-
*
|
|
784
|
+
*
|
|
785
|
+
* Healthy means both: the HTTP check (or the no-port settle) passes, AND PM2
|
|
786
|
+
* keeps it online without increasing its restart count over a short
|
|
787
|
+
* confirmation window. The second half is what catches a release that answers
|
|
788
|
+
* a probe and then crashes at boot.
|
|
789
|
+
*
|
|
790
|
+
* @returns {Promise<{ ok: boolean, detail: string, crash: boolean, reason: string|null }>}
|
|
645
791
|
*/
|
|
646
792
|
async function waitHealthy({ name, port, healthPath, strict, timeoutS, pm2Path, d }) {
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
793
|
+
let baseline = null;
|
|
794
|
+
let last = 'no response yet';
|
|
795
|
+
|
|
796
|
+
/** @returns {string|null} A boot-crash reason, or null */
|
|
797
|
+
const observe = (observed) => {
|
|
798
|
+
const reason = bootCrashReason(observed, baseline, { port });
|
|
799
|
+
if (observed.read && observed.apps.length > 0 && baseline === null) {
|
|
800
|
+
baseline = new Map();
|
|
801
|
+
observed.apps.forEach((app, i) => {
|
|
802
|
+
baseline.set(app.pmId ?? i, Number.isFinite(app.restarts) ? app.restarts : 0);
|
|
803
|
+
});
|
|
804
|
+
}
|
|
805
|
+
return reason;
|
|
651
806
|
};
|
|
652
807
|
|
|
808
|
+
const failOrNull = (reason) => (reason
|
|
809
|
+
? { ok: false, crash: true, detail: reason, reason }
|
|
810
|
+
: null);
|
|
811
|
+
|
|
653
812
|
if (!port) {
|
|
654
813
|
// Nothing to ask over HTTP: the app has to stay online, without PM2
|
|
655
814
|
// restarting it, for a few seconds.
|
|
656
815
|
await d.sleep(Math.min(2000, NO_PORT_SETTLE_MS));
|
|
657
|
-
const early = await
|
|
816
|
+
const early = await readPm2Status(name, pm2Path, d);
|
|
817
|
+
const earlyCrash = failOrNull(observe(early));
|
|
818
|
+
if (earlyCrash) return earlyCrash;
|
|
819
|
+
|
|
658
820
|
await d.sleep(NO_PORT_SETTLE_MS - 2000);
|
|
659
|
-
const late = await
|
|
660
|
-
|
|
661
|
-
if (
|
|
662
|
-
|
|
663
|
-
|
|
821
|
+
const late = await readPm2Status(name, pm2Path, d);
|
|
822
|
+
const lateCrash = failOrNull(observe(late));
|
|
823
|
+
if (lateCrash) return lateCrash;
|
|
824
|
+
|
|
825
|
+
if (late.apps.length === 0) return { ok: false, crash: true, detail: 'crashed at boot: PM2 no longer lists the process', reason: 'crashed at boot: PM2 no longer lists the process' };
|
|
826
|
+
const notOnline = late.apps.find((a) => a.status !== 'online');
|
|
827
|
+
if (notOnline) {
|
|
828
|
+
const reason = `crashed at boot: process is "${notOnline.status}", not online`;
|
|
829
|
+
return { ok: false, crash: true, detail: reason, reason };
|
|
664
830
|
}
|
|
665
|
-
return {
|
|
831
|
+
return {
|
|
832
|
+
ok: true,
|
|
833
|
+
crash: false,
|
|
834
|
+
reason: null,
|
|
835
|
+
detail: 'The process stayed online (no port is configured, so no HTTP check was made).',
|
|
836
|
+
};
|
|
666
837
|
}
|
|
667
838
|
|
|
668
839
|
const deadline = d.now() + timeoutS * 1000;
|
|
669
|
-
let
|
|
840
|
+
let answeredOnce = false;
|
|
670
841
|
|
|
671
842
|
for (;;) {
|
|
672
843
|
const res = await d.probe(port, healthPath);
|
|
673
844
|
if (res.ok && isHealthyStatus(res.status, strict)) {
|
|
674
|
-
|
|
845
|
+
if (answeredOnce) {
|
|
846
|
+
const observed = await readPm2Status(name, pm2Path, d);
|
|
847
|
+
const crash = failOrNull(observe(observed));
|
|
848
|
+
if (crash) return crash;
|
|
849
|
+
// A PM2 status that could not be read is not evidence of a crash; the
|
|
850
|
+
// HTTP answer is what we have.
|
|
851
|
+
if (!observed.read || (observed.apps.length > 0 && observed.apps.every((a) => a.status === 'online'))) {
|
|
852
|
+
return { ok: true, crash: false, reason: null, detail: `GET ${healthPath} on port ${port} answered ${res.status}, and PM2 stayed online.` };
|
|
853
|
+
}
|
|
854
|
+
} else {
|
|
855
|
+
const first = await readPm2Status(name, pm2Path, d);
|
|
856
|
+
const crash = failOrNull(observe(first));
|
|
857
|
+
if (crash) return crash;
|
|
858
|
+
answeredOnce = true;
|
|
859
|
+
// Keep watching for one short window: an app that answers and then
|
|
860
|
+
// crash-loops must not be accepted on the strength of that first answer.
|
|
861
|
+
const remaining = Math.max(0, deadline - d.now());
|
|
862
|
+
await d.sleep(Math.min(BOOT_CONFIRM_MS, remaining));
|
|
863
|
+
continue;
|
|
864
|
+
}
|
|
675
865
|
}
|
|
676
866
|
last = res.ok ? `it answered ${res.status}` : `no answer (${res.error})`;
|
|
677
867
|
|
|
678
|
-
|
|
679
|
-
const
|
|
680
|
-
if (
|
|
681
|
-
|
|
868
|
+
const observed = await readPm2Status(name, pm2Path, d);
|
|
869
|
+
const crash = failOrNull(observe(observed));
|
|
870
|
+
if (crash) return crash;
|
|
871
|
+
|
|
872
|
+
if (d.now() >= deadline) {
|
|
873
|
+
// A process that never reached "online" is a boot crash, not merely a
|
|
874
|
+
// slow health endpoint.
|
|
875
|
+
const notOnline = observed.read ? observed.apps.find((a) => a.status !== 'online') : null;
|
|
876
|
+
if (notOnline) {
|
|
877
|
+
const reason = `crashed at boot: process is "${notOnline.status}", not online`;
|
|
878
|
+
return { ok: false, crash: true, detail: reason, reason };
|
|
879
|
+
}
|
|
880
|
+
const detail = `No healthy response from GET ${healthPath} on port ${port} within ${timeoutS}s (${last}).`;
|
|
881
|
+
return { ok: false, crash: false, detail, reason: `health check failed: ${detail}` };
|
|
682
882
|
}
|
|
683
|
-
|
|
684
|
-
if (d.now() >= deadline) break;
|
|
685
883
|
await d.sleep(HEALTH_POLL_MS);
|
|
686
884
|
}
|
|
885
|
+
}
|
|
687
886
|
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
887
|
+
/**
|
|
888
|
+
* Reads the tail of a crashed release's PM2 out+error log, with the app's
|
|
889
|
+
* secret values removed.
|
|
890
|
+
*
|
|
891
|
+
* @returns {Promise<string[]>} At most BOOT_LOG_LINES lines
|
|
892
|
+
*/
|
|
893
|
+
async function captureBootLog(name, app, pm2Path, d) {
|
|
894
|
+
if (typeof d.runner.logs !== 'function') return [];
|
|
895
|
+
try {
|
|
896
|
+
const read = await d.runner.logs(name, { lines: BOOT_LOG_LINES, run: d.run, pm2Path });
|
|
897
|
+
const text = Array.isArray(read?.lines) ? read.lines.join('\n') : String(read?.output ?? '');
|
|
898
|
+
return text
|
|
899
|
+
.replace(/\r\n/g, '\n')
|
|
900
|
+
.split('\n')
|
|
901
|
+
.filter((line) => line.trim() !== '')
|
|
902
|
+
.slice(-BOOT_LOG_LINES)
|
|
903
|
+
.map((line) => redactSecrets(line, app.env ?? {}, app.auth?.token ?? null));
|
|
904
|
+
} catch {
|
|
905
|
+
return [];
|
|
906
|
+
}
|
|
692
907
|
}
|
|
693
908
|
|
|
694
|
-
/**
|
|
695
|
-
|
|
909
|
+
/**
|
|
910
|
+
* Starts a release under PM2, or reloads it when both the new release and the
|
|
911
|
+
* running app are in cluster mode.
|
|
912
|
+
*
|
|
913
|
+
* `delete` rather than `restart` is the ordinary path: the working directory
|
|
914
|
+
* and the environment belong to the release, and PM2 keeps the old ones across
|
|
915
|
+
* a restart. Deleting a name PM2 does not know fails harmlessly on a first
|
|
916
|
+
* deployment. Cluster-to-cluster is the exception — PM2 reload overlaps the
|
|
917
|
+
* workers, which is the zero-downtime handover; a first deployment or a
|
|
918
|
+
* fork↔cluster switch cannot overlap and takes the delete/start path.
|
|
919
|
+
*
|
|
920
|
+
* @param {{ execMode?: 'fork'|'cluster' }} [options]
|
|
921
|
+
*/
|
|
922
|
+
async function switchTo(dir, name, pm2Path, d, log, { execMode = 'fork' } = {}) {
|
|
696
923
|
const config = path.join(dir, PM2_CONFIG_FILE);
|
|
697
924
|
if (!fs.existsSync(config)) return { ok: false, output: `${PM2_CONFIG_FILE} is missing from ${dir}.` };
|
|
698
925
|
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
926
|
+
if (execMode === 'cluster' && typeof d.runner.reloadConfig === 'function') {
|
|
927
|
+
const current = await readPm2Status(name, pm2Path, d);
|
|
928
|
+
const running = current.read && current.apps.length > 0;
|
|
929
|
+
const allCluster = running && current.apps.every((a) => a.execMode === 'cluster');
|
|
930
|
+
if (allCluster) {
|
|
931
|
+
log.line(`$ pm2 reload ${PM2_CONFIG_FILE} (cluster mode, zero downtime)`);
|
|
932
|
+
const reloaded = await d.runner.reloadConfig(config, { run: d.run, pm2Path });
|
|
933
|
+
if (!reloaded.ok) log.line(reloaded.output || 'pm2 reload failed with no output');
|
|
934
|
+
return reloaded;
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
|
|
702
938
|
await d.runner.del(name, { run: d.run, pm2Path });
|
|
703
939
|
const started = await d.runner.startConfig(config, { run: d.run, pm2Path });
|
|
704
940
|
if (!started.ok) log.line(started.output || 'pm2 start failed with no output');
|
|
@@ -706,7 +942,12 @@ async function switchTo(dir, name, pm2Path, d, log) {
|
|
|
706
942
|
}
|
|
707
943
|
|
|
708
944
|
async function notify(app, run, d) {
|
|
709
|
-
|
|
945
|
+
// A boot crash is a failed run, but when the previous release is serving
|
|
946
|
+
// again the notification should say so rather than only "failed".
|
|
947
|
+
const outcome = run.rollbackSucceeded === true && run.status === RUN_STATUS.FAILED
|
|
948
|
+
? RUN_STATUS.ROLLED_BACK
|
|
949
|
+
: run.status;
|
|
950
|
+
const event = createDeployEvent(app.id, outcome, {
|
|
710
951
|
sha: run.sha,
|
|
711
952
|
reason: run.error,
|
|
712
953
|
rollback: run.trigger === 'rollback',
|
|
@@ -766,6 +1007,12 @@ export function startDeployment(appId, { trigger = 'manual', ref = null, actor =
|
|
|
766
1007
|
error: null,
|
|
767
1008
|
warnings: [],
|
|
768
1009
|
rolledBackTo: null,
|
|
1010
|
+
/** A short human reason for a boot crash or health failure, else null. */
|
|
1011
|
+
reason: null,
|
|
1012
|
+
/** Redacted PM2 out+error tail for a boot crash; empty otherwise. */
|
|
1013
|
+
logTail: [],
|
|
1014
|
+
/** true/false once an automatic rollback was attempted, null when none was. */
|
|
1015
|
+
rollbackSucceeded: null,
|
|
769
1016
|
};
|
|
770
1017
|
writeRun(run);
|
|
771
1018
|
|
|
@@ -811,18 +1058,34 @@ async function executeDeployment(app, run, d) {
|
|
|
811
1058
|
};
|
|
812
1059
|
|
|
813
1060
|
/** Runs the user's commands for one step, in the release directory. */
|
|
814
|
-
const userCommands = async (step, title, commands, env) => {
|
|
1061
|
+
const userCommands = async (step, title, commands, env, { hint = null, includeOutput = false } = {}) => {
|
|
815
1062
|
if (!commands || commands.length === 0) return;
|
|
816
1063
|
setStep(step, title);
|
|
817
1064
|
for (const command of commands) {
|
|
818
1065
|
log.line(`$ ${command}`);
|
|
1066
|
+
let tail = '';
|
|
819
1067
|
const r = await d.runCommandLine(command, {
|
|
820
|
-
cwd: dir,
|
|
1068
|
+
cwd: dir,
|
|
1069
|
+
env,
|
|
1070
|
+
timeout: STEP_TIMEOUT_MS,
|
|
1071
|
+
onData: (c) => {
|
|
1072
|
+
log.output(c);
|
|
1073
|
+
if (includeOutput) {
|
|
1074
|
+
tail += String(c);
|
|
1075
|
+
if (tail.length > 4000) tail = tail.slice(-4000);
|
|
1076
|
+
}
|
|
1077
|
+
},
|
|
821
1078
|
});
|
|
822
1079
|
log.flush();
|
|
823
1080
|
if (!r.success) {
|
|
824
1081
|
const why = r.exitCode === null ? (r.stderr || 'it was stopped') : `exit code ${r.exitCode}`;
|
|
825
|
-
|
|
1082
|
+
const lines = includeOutput
|
|
1083
|
+
? tail.replace(/\r\n/g, '\n').split('\n').map((l) => l.trim()).filter(Boolean).slice(-5)
|
|
1084
|
+
.map((l) => redactSecrets(l, app.env ?? {}, app.auth?.token ?? null))
|
|
1085
|
+
: [];
|
|
1086
|
+
const detail = `${title} failed: "${command}" — ${why}${lines.length ? `\n${lines.join('\n')}` : ''}${hint ? `\n${hint}` : ''}`;
|
|
1087
|
+
if (hint) log.line(`hint: ${hint}`);
|
|
1088
|
+
throw new StepError(step, detail);
|
|
826
1089
|
}
|
|
827
1090
|
}
|
|
828
1091
|
};
|
|
@@ -866,10 +1129,16 @@ async function executeDeployment(app, run, d) {
|
|
|
866
1129
|
|
|
867
1130
|
const { settings, sources } = project;
|
|
868
1131
|
log.line(project.ezLocation ? `ez.json found at ${project.ezLocation}` : 'no ez.json — using package.json defaults');
|
|
869
|
-
for (const field of ['install', 'build', 'start', 'port', 'healthPath']) {
|
|
1132
|
+
for (const field of ['install', 'build', 'migrate', 'start', 'runtime', 'port', 'healthPath', 'execMode', 'instances']) {
|
|
870
1133
|
const value = Array.isArray(settings[field]) ? (settings[field].join(' && ') || '(none)') : (settings[field] ?? '(none)');
|
|
871
1134
|
log.line(` ${field.padEnd(10)} ${value} [${sources[field]}]`);
|
|
872
1135
|
}
|
|
1136
|
+
if (settings.steps.length > 0) {
|
|
1137
|
+
const shown = settings.steps
|
|
1138
|
+
.map((s, i) => `${i + 1}.${s.kind}${s.enabled === false ? ' (disabled)' : ''}: ${s.run}`)
|
|
1139
|
+
.join(' | ');
|
|
1140
|
+
log.line(` steps ${shown} [${sources.steps}]`);
|
|
1141
|
+
}
|
|
873
1142
|
if (settings.persist.length > 0) {
|
|
874
1143
|
log.line(` persist ${settings.persist.join(', ')} [${sources.persist}]`);
|
|
875
1144
|
}
|
|
@@ -883,10 +1152,30 @@ async function executeDeployment(app, run, d) {
|
|
|
883
1152
|
const env = appEnvironment(app, settings, run.releaseId);
|
|
884
1153
|
const strict = sources.healthPath !== 'default';
|
|
885
1154
|
|
|
886
|
-
// ── 3. pre → install
|
|
1155
|
+
// ── 3. pre → steps (install/build/migrate/custom) ───────────────────────
|
|
887
1156
|
await userCommands('pre', 'Pre-deploy hook', settings.pre, env);
|
|
888
|
-
|
|
889
|
-
|
|
1157
|
+
for (let i = 0; i < settings.steps.length; i++) {
|
|
1158
|
+
const step = settings.steps[i];
|
|
1159
|
+
const number = i + 1;
|
|
1160
|
+
if (step.enabled === false) {
|
|
1161
|
+
log.line(`Skipping step ${number} (${step.kind}): disabled.`);
|
|
1162
|
+
continue;
|
|
1163
|
+
}
|
|
1164
|
+
// The run record's `step` is the kind for the three steps that have
|
|
1165
|
+
// special behaviour, and "step N" for a custom one.
|
|
1166
|
+
const stepId = step.kind === 'custom' ? `step ${number}` : step.kind;
|
|
1167
|
+
const title = step.kind === 'install'
|
|
1168
|
+
? 'Install'
|
|
1169
|
+
: step.kind === 'build'
|
|
1170
|
+
? 'Build'
|
|
1171
|
+
: step.kind === 'migrate'
|
|
1172
|
+
? 'Migrate'
|
|
1173
|
+
: `Step ${number}`;
|
|
1174
|
+
await userCommands(stepId, title, [step.run], env, {
|
|
1175
|
+
hint: step.kind === 'migrate' ? MIGRATE_HINT : null,
|
|
1176
|
+
includeOutput: step.kind === 'migrate',
|
|
1177
|
+
});
|
|
1178
|
+
}
|
|
890
1179
|
verifyPersistentLinks(app.id, dir, settings.persist);
|
|
891
1180
|
|
|
892
1181
|
// ── 4. switch ───────────────────────────────────────────────────────────
|
|
@@ -902,12 +1191,14 @@ async function executeDeployment(app, run, d) {
|
|
|
902
1191
|
fs.writeFileSync(path.join(dir, 'ez.release.json'), JSON.stringify({
|
|
903
1192
|
sha: run.sha, ref: run.ref, port: settings.port, healthPath: settings.healthPath, strict,
|
|
904
1193
|
persist: settings.persist,
|
|
1194
|
+
execMode: settings.execMode,
|
|
1195
|
+
instances: settings.instances,
|
|
905
1196
|
deployedAt: new Date(d.now()).toISOString(),
|
|
906
1197
|
}, null, 2));
|
|
907
1198
|
|
|
908
|
-
log.line(`$ pm2 delete ${name}; pm2 start ${PM2_CONFIG_FILE}`);
|
|
1199
|
+
log.line(`$ pm2 delete ${name}; pm2 start ${PM2_CONFIG_FILE}${settings.execMode === 'cluster' ? ' (cluster mode; a running cluster app is reloaded instead)' : ''}`);
|
|
909
1200
|
switched = true;
|
|
910
|
-
const started = await switchTo(dir, name, pm2Path, d, log);
|
|
1201
|
+
const started = await switchTo(dir, name, pm2Path, d, log, { execMode: settings.execMode });
|
|
911
1202
|
if (!started.ok) throw new StepError('start', 'PM2 could not start the new release.');
|
|
912
1203
|
|
|
913
1204
|
// ── 5. health ───────────────────────────────────────────────────────────
|
|
@@ -917,7 +1208,9 @@ async function executeDeployment(app, run, d) {
|
|
|
917
1208
|
timeoutS: settings.healthTimeout, pm2Path, d,
|
|
918
1209
|
});
|
|
919
1210
|
log.line(health.detail);
|
|
920
|
-
if (!health.ok)
|
|
1211
|
+
if (!health.ok) {
|
|
1212
|
+
throw new StepError('health', health.detail, { bootCrash: health.crash, reason: health.reason });
|
|
1213
|
+
}
|
|
921
1214
|
|
|
922
1215
|
// ── 6. it is live ───────────────────────────────────────────────────────
|
|
923
1216
|
writeState(app.id, { current: run.releaseId, previous: before.current });
|
|
@@ -952,25 +1245,40 @@ async function executeDeployment(app, run, d) {
|
|
|
952
1245
|
log.line('Deployment succeeded.');
|
|
953
1246
|
} catch (err) {
|
|
954
1247
|
log.flush();
|
|
1248
|
+
const bootCrash = err?.bootCrash === true;
|
|
955
1249
|
run.error = err?.message ?? String(err);
|
|
956
1250
|
run.step = err?.step ?? run.step;
|
|
1251
|
+
run.reason = err?.reason ?? null;
|
|
957
1252
|
log.line(`ERROR: ${run.error}`);
|
|
958
1253
|
run.status = RUN_STATUS.FAILED;
|
|
959
1254
|
|
|
1255
|
+
// Read the crashed release's own PM2 log BEFORE a rollback reuses the
|
|
1256
|
+
// same PM2 name — afterwards these lines belong to the restored release.
|
|
1257
|
+
if (bootCrash) {
|
|
1258
|
+
run.logTail = await captureBootLog(name, app, pm2Path, d);
|
|
1259
|
+
if (run.logTail.length > 0) {
|
|
1260
|
+
log.step(`PM2 log (last ${run.logTail.length} lines, secrets redacted)`);
|
|
1261
|
+
for (const line of run.logTail) log.line(line);
|
|
1262
|
+
} else {
|
|
1263
|
+
log.line('PM2 produced no log lines for the crashed release.');
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
|
|
960
1267
|
if (!switched) {
|
|
961
1268
|
// Nothing that was running has been touched. The half-built directory
|
|
962
1269
|
// is of no use to a rollback and can be large; the log is what explains
|
|
963
1270
|
// the failure.
|
|
964
1271
|
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
1272
|
+
run.logTail = [];
|
|
965
1273
|
log.line('The running release was not touched.');
|
|
966
1274
|
} else if (before.current && app.autoRollback !== false) {
|
|
967
1275
|
log.step(`Rolling back to ${before.current}`);
|
|
968
1276
|
let restored = false;
|
|
969
1277
|
try {
|
|
970
1278
|
const previousDir = releaseDir(app.id, before.current);
|
|
971
|
-
const
|
|
1279
|
+
const meta = releaseHealth(previousDir);
|
|
1280
|
+
const back = await switchTo(previousDir, name, pm2Path, d, log, { execMode: meta.execMode });
|
|
972
1281
|
if (back.ok) {
|
|
973
|
-
const meta = releaseHealth(previousDir);
|
|
974
1282
|
const health = await waitHealthy({
|
|
975
1283
|
name, port: meta.port, healthPath: meta.healthPath, strict: meta.strict,
|
|
976
1284
|
timeoutS: ROLLBACK_HEALTH_TIMEOUT_S, pm2Path, d,
|
|
@@ -982,8 +1290,13 @@ async function executeDeployment(app, run, d) {
|
|
|
982
1290
|
log.line(`Rollback error: ${rollbackErr.message}`);
|
|
983
1291
|
}
|
|
984
1292
|
|
|
1293
|
+
run.rollbackSucceeded = restored;
|
|
985
1294
|
if (restored) {
|
|
986
|
-
run
|
|
1295
|
+
// A boot crash stays a failed run — what was deployed never worked —
|
|
1296
|
+
// but the fact that the previous release is serving again is recorded
|
|
1297
|
+
// rather than hidden. A plain health-check failure keeps the older
|
|
1298
|
+
// `rolled_back` status.
|
|
1299
|
+
if (!bootCrash) run.status = RUN_STATUS.ROLLED_BACK;
|
|
987
1300
|
run.rolledBackTo = before.current;
|
|
988
1301
|
log.line(`The previous release (${before.current}) is running again.`);
|
|
989
1302
|
} else {
|
|
@@ -993,10 +1306,11 @@ async function executeDeployment(app, run, d) {
|
|
|
993
1306
|
} else {
|
|
994
1307
|
// No release to return to (a first deployment) or rollback is switched
|
|
995
1308
|
// off. Remove the broken process so PM2 does not restart it in a loop.
|
|
1309
|
+
run.rollbackSucceeded = null;
|
|
996
1310
|
try { await d.runner.del(name, { run: d.run, pm2Path }); } catch { /* ignore */ }
|
|
997
1311
|
log.line(before.current
|
|
998
|
-
? 'Automatic rollback is switched off for this app; the process was stopped.'
|
|
999
|
-
: 'There is no earlier release to roll back to
|
|
1312
|
+
? 'Automatic rollback is switched off for this app; the process was stopped. Fix the failure and deploy again.'
|
|
1313
|
+
: 'There is no earlier release to roll back to, so the new release was stopped. Fix the failure and deploy again.');
|
|
1000
1314
|
}
|
|
1001
1315
|
}
|
|
1002
1316
|
|
|
@@ -1058,6 +1372,9 @@ export function startRollback(appId, { releaseId = null, actor = null, deps = {}
|
|
|
1058
1372
|
error: null,
|
|
1059
1373
|
warnings: [],
|
|
1060
1374
|
rolledBackTo: null,
|
|
1375
|
+
reason: null,
|
|
1376
|
+
logTail: [],
|
|
1377
|
+
rollbackSucceeded: null,
|
|
1061
1378
|
};
|
|
1062
1379
|
writeRun(run);
|
|
1063
1380
|
|
|
@@ -1073,7 +1390,7 @@ export function startRollback(appId, { releaseId = null, actor = null, deps = {}
|
|
|
1073
1390
|
pm2Path = pm2.path ?? 'pm2';
|
|
1074
1391
|
|
|
1075
1392
|
log.step(`Starting release ${target}`);
|
|
1076
|
-
const started = await switchTo(targetDir, name, pm2Path, d, log);
|
|
1393
|
+
const started = await switchTo(targetDir, name, pm2Path, d, log, { execMode: meta.execMode });
|
|
1077
1394
|
if (!started.ok) throw new StepError('start', 'PM2 could not start that release.');
|
|
1078
1395
|
|
|
1079
1396
|
run.step = 'health';
|
|
@@ -1084,7 +1401,7 @@ export function startRollback(appId, { releaseId = null, actor = null, deps = {}
|
|
|
1084
1401
|
timeoutS: ROLLBACK_HEALTH_TIMEOUT_S, pm2Path, d,
|
|
1085
1402
|
});
|
|
1086
1403
|
log.line(health.detail);
|
|
1087
|
-
if (!health.ok) throw new StepError('health', health.detail);
|
|
1404
|
+
if (!health.ok) throw new StepError('health', health.detail, { bootCrash: health.crash, reason: health.reason });
|
|
1088
1405
|
|
|
1089
1406
|
writeState(appId, { current: target, previous: before.current });
|
|
1090
1407
|
await d.runner.save({ run: d.run, pm2Path });
|
|
@@ -1094,13 +1411,15 @@ export function startRollback(appId, { releaseId = null, actor = null, deps = {}
|
|
|
1094
1411
|
} catch (err) {
|
|
1095
1412
|
run.status = RUN_STATUS.FAILED;
|
|
1096
1413
|
run.error = err?.message ?? String(err);
|
|
1414
|
+
run.reason = err?.reason ?? null;
|
|
1097
1415
|
run.step = err?.step ?? run.step;
|
|
1098
1416
|
log.line(`ERROR: ${run.error}`);
|
|
1099
1417
|
|
|
1100
1418
|
// Put back what was running, so a failed rollback is not also an outage.
|
|
1101
1419
|
if (before.current) {
|
|
1102
1420
|
try {
|
|
1103
|
-
const
|
|
1421
|
+
const restoreDir = releaseDir(appId, before.current);
|
|
1422
|
+
const back = await switchTo(restoreDir, name, pm2Path, d, log, { execMode: releaseHealth(restoreDir).execMode });
|
|
1104
1423
|
log.line(back.ok
|
|
1105
1424
|
? `Release ${before.current} was started again.`
|
|
1106
1425
|
: `Release ${before.current} could NOT be started again. The app may be down.`);
|