easy-devops 1.3.6 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +1 -0
  2. package/dist/ui/assets/index-Bu7af-Kj.css +1 -0
  3. package/dist/ui/assets/index-gndgYe3K.js +532 -0
  4. package/dist/ui/assets/socket-Bv0wCEgi.js +65 -0
  5. package/dist/ui/index.html +2 -2
  6. package/package.json +2 -2
  7. package/src/cli/index.js +8 -0
  8. package/src/cli/managers/domain-manager.js +37 -6
  9. package/src/cli/managers/nginx-manager.js +14 -5
  10. package/src/cli/managers/node-manager.js +19 -13
  11. package/src/cli/managers/ssl-manager.js +89 -26
  12. package/src/cli/menu.js +8 -2
  13. package/src/cli/menus/cleanup.js +4 -2
  14. package/src/cli/menus/dashboard.js +22 -15
  15. package/src/cli/menus/hub.js +381 -0
  16. package/src/cli/menus/runtime.js +12 -0
  17. package/src/cli/menus/service.js +28 -10
  18. package/src/cli/menus/settings.js +46 -12
  19. package/src/cli/menus/uninstall.js +13 -5
  20. package/src/cli/menus/update.js +24 -14
  21. package/src/cli/ui/page.js +123 -0
  22. package/src/core/cert-operations.js +34 -4
  23. package/src/core/cert-reader.js +67 -22
  24. package/src/core/cert-renewal.js +65 -9
  25. package/src/core/cert-suggest.js +120 -0
  26. package/src/core/cleanup.js +221 -1
  27. package/src/core/cloudflare-ips.js +161 -0
  28. package/src/core/daemon/ipc.js +1 -1
  29. package/src/core/daemon/lock.js +296 -9
  30. package/src/core/dashboardProcess.js +134 -6
  31. package/src/core/deploy/apps.js +36 -62
  32. package/src/core/deploy/deployKey.js +331 -0
  33. package/src/core/deploy/domainLink.js +280 -23
  34. package/src/core/deploy/engine.js +382 -63
  35. package/src/core/deploy/ezfile.js +496 -29
  36. package/src/core/deploy/githubApi.js +100 -13
  37. package/src/core/deploy/hubCommands.js +158 -18
  38. package/src/core/deploy/status.js +28 -1
  39. package/src/core/domains-db.js +1 -0
  40. package/src/core/hub/agent.js +238 -32
  41. package/src/core/hub/deployProgress.js +411 -0
  42. package/src/core/hubConnect.js +205 -0
  43. package/src/core/hubPermissions.js +48 -10
  44. package/src/core/jobs/bus.js +1 -1
  45. package/src/core/jobs/owner.js +268 -6
  46. package/src/core/nginx-conf-generator.js +124 -19
  47. package/src/core/notifications/claim.js +26 -3
  48. package/src/core/permissions.js +277 -59
  49. package/src/core/platform.js +48 -2
  50. package/src/core/runner.js +96 -40
  51. package/src/core/ssl/preflight.js +548 -0
  52. package/src/core/ssl/progress.js +196 -0
  53. package/src/core/ssl-issuer.js +502 -148
  54. package/src/core/validators.js +24 -0
  55. package/src/dashboard/controllers/authController.js +12 -5
  56. package/src/dashboard/controllers/deploymentsController.js +76 -11
  57. package/src/dashboard/controllers/domainsController.js +35 -1
  58. package/src/dashboard/controllers/hubController.js +4 -3
  59. package/src/dashboard/controllers/settingsController.js +73 -68
  60. package/src/dashboard/controllers/sslController.js +135 -15
  61. package/src/dashboard/public/js/app.js +42 -7
  62. package/src/dashboard/routes/deployments.js +2 -0
  63. package/src/dashboard/routes/domains.js +1 -0
  64. package/src/dashboard/server.js +42 -7
  65. package/src/dashboard/socketManager.js +19 -25
  66. package/src/dashboard/views/partials/settings-panel.ejs +29 -10
  67. package/dist/ui/assets/index-BUpPxWmp.css +0 -1
  68. package/dist/ui/assets/index-DD0JjK0Q.js +0 -475
  69. package/dist/ui/assets/socket-4tfdap-e.js +0 -65
@@ -7,14 +7,27 @@
7
7
  *
8
8
  * 1. fetch a fresh checkout into releases/<id> — never over the live one
9
9
  * 2. settings ez.json + package.json + the app's overrides (ezfile.js)
10
- * 3. pre hook → install → build all in the new directory
10
+ * 3. pre hook → the steps list (install/build/migrate/custom) in order
11
11
  * 4. switch stop the old process, start the new one under PM2
12
12
  * 5. health the app must answer (or stay up) before it counts
13
13
  * 6. post hook, save the PM2 list, prune old releases
14
14
  *
15
- * Nothing before step 4 touches what is running, so a broken build costs
16
- * nothing. If step 4 or 5 fails and there is a previous release, it is started
17
- * again — the same files that were running before, not a rebuild.
15
+ * Every step runs in the new release with the build's environment, before
16
+ * anything is started. `migrate` keeps its forward-only hint: rolling the code
17
+ * back does not undo a migration that already ran, which is why the hint on a
18
+ * failure says to fix the migration rather than expect the rollback to undo it.
19
+ * A disabled step is logged as skipped.
20
+ *
21
+ * Nothing before step 4 touches what is running, so a broken build or a failed
22
+ * migration costs nothing. If step 4 or 5 fails and there is a previous
23
+ * release, it is started again — the same files that were running before, not a
24
+ * rebuild.
25
+ *
26
+ * Step 5 watches PM2 as well as HTTP. A release that answers once and then
27
+ * crash-loops, or that PM2 reports errored/stopped, is a boot crash: the
28
+ * previous release is restored, and the run records why, the last PM2 log
29
+ * lines (with the app's environment values redacted) and whether that rollback
30
+ * succeeded. A first release has nothing to return to, so it is stopped.
18
31
  *
19
32
  * ── What this does not do ────────────────────────────────────────────────────
20
33
  *
@@ -45,7 +58,8 @@ import { createDeployEvent } from '../events.js';
45
58
  import { sendNotification as defaultSendNotification } from '../notifier.js';
46
59
  import { publish } from '../jobs/bus.js';
47
60
 
48
- import { getApp, deleteApp, pm2Name, AUTH_TYPES, deployKeyPath, validateRef, setAppDir } from './apps.js';
61
+ import { getApp, deleteApp, pm2Name, AUTH_TYPES, validateRef, setAppDir } from './apps.js';
62
+ import { sshCommand } from './deployKey.js';
49
63
  import { deleteLinkedDomain, syncLinkedDomain } from './domainLink.js';
50
64
  import {
51
65
  EZ_FILE_LOCATIONS, parseEzFile, resolveSettings, parseStartCommand,
@@ -66,6 +80,20 @@ const HEALTH_POLL_MS = 2000;
66
80
  /** Without a port there is nothing to ask, so the app only has to stay up this long. */
67
81
  const NO_PORT_SETTLE_MS = 8000;
68
82
  const ROLLBACK_HEALTH_TIMEOUT_S = 30;
83
+ /**
84
+ * How long a healthy answer must also stay online without a PM2 restart.
85
+ * PM2's own `min_uptime` is 5 s, so a release that dies inside that window is
86
+ * one PM2 itself considers unstable; matching it keeps the two judgements
87
+ * aligned.
88
+ */
89
+ const BOOT_CONFIRM_MS = 5000;
90
+ /** PM2 out+error lines kept in the run record when a release crashes at boot. */
91
+ const BOOT_LOG_LINES = 40;
92
+
93
+ const MIGRATE_HINT =
94
+ 'Migrations are forward-only: rolling the code back cannot undo a migration that already ran. ' +
95
+ 'Fix the migration, keep it backward-compatible for one release, and deploy again. ' +
96
+ 'The release that was running was not touched.';
69
97
 
70
98
  export const RUN_STATUS = Object.freeze({
71
99
  RUNNING: 'running',
@@ -76,9 +104,11 @@ export const RUN_STATUS = Object.freeze({
76
104
 
77
105
  /** A failure with the step it happened in, so the run record can say where. */
78
106
  class StepError extends Error {
79
- constructor(step, message) {
107
+ constructor(step, message, { bootCrash = false, reason = null } = {}) {
80
108
  super(message);
81
109
  this.step = step;
110
+ this.bootCrash = bootCrash;
111
+ this.reason = reason;
82
112
  }
83
113
  }
84
114
 
@@ -118,6 +148,31 @@ export function isHealthyStatus(status, strict) {
118
148
  return strict ? status >= 200 && status < 400 : status >= 100 && status < 500;
119
149
  }
120
150
 
151
+ /**
152
+ * Removes the app's secret values from text that is about to leave the engine.
153
+ *
154
+ * PM2's own logs are the app's output, and an app that prints its configuration
155
+ * — or a crash dump that includes it — would otherwise put a database URL or a
156
+ * token into a run record the dashboard and the Hub can read. Every non-empty
157
+ * value of an environment variable on the app is replaced, longest first, so a
158
+ * secret that contains another one is not half-redacted. The auth token is
159
+ * included for the same reason even though it lives outside `env`.
160
+ *
161
+ * @param {string} text
162
+ * @param {Object<string, *>} [env]
163
+ * @param {string|null} [token]
164
+ * @returns {string}
165
+ */
166
+ export function redactSecrets(text, env = {}, token = null) {
167
+ let out = String(text ?? '');
168
+ const values = [...Object.values(env ?? {}), token]
169
+ .filter((v) => typeof v === 'string' && v.length > 0)
170
+ .sort((a, b) => b.length - a.length);
171
+
172
+ for (const value of values) out = out.split(value).join('***');
173
+ return out;
174
+ }
175
+
121
176
  // ─── The engine ───────────────────────────────────────────────────────────────
122
177
 
123
178
  function resolveDeps(deps = {}) {
@@ -192,13 +247,17 @@ function gitAuth(app) {
192
247
  }
193
248
 
194
249
  if (app.auth?.type === AUTH_TYPES.DEPLOY_KEY) {
195
- const key = deployKeyPath(app.id);
196
- if (!key) throw new StepError('fetch', 'This app uses a deploy key, but none has been generated yet.');
197
- // Forward slashes: git runs this through its own sh, where a backslash in
198
- // a Windows path would be read as an escape.
199
- const keyArg = key.replace(/\\/g, '/');
200
- env.GIT_SSH_COMMAND =
201
- `ssh -i "${keyArg}" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new`;
250
+ // github.com is checked against GitHub's pinned host keys; see deployKey.js.
251
+ let sshCmd;
252
+ try {
253
+ sshCmd = sshCommand(app.id, app.repoUrl);
254
+ } catch (err) {
255
+ throw new StepError('fetch', err.message);
256
+ }
257
+ if (!sshCmd) {
258
+ throw new StepError('fetch', 'This app uses a deploy key, but none has been generated yet. Generate it in the app settings, add it to the repository, then deploy again.');
259
+ }
260
+ env.GIT_SSH_COMMAND = sshCmd;
202
261
  }
203
262
 
204
263
  return { env, flags: '', display: '' };
@@ -238,7 +297,12 @@ export function loadProjectSettings(dir, overrides = {}) {
238
297
  };
239
298
 
240
299
  const resolved = resolveSettings({ ez, pkg, files, overrides });
241
- return { ...resolved, warnings, ezLocation };
300
+ return {
301
+ ...resolved,
302
+ // ez.json warnings plus the runtime warnings (cluster mode's stateless note).
303
+ warnings: [...warnings, ...(resolved.warnings ?? [])],
304
+ ezLocation,
305
+ };
242
306
  }
243
307
 
244
308
  // ─── Persistent paths ────────────────────────────────────────────────────────
@@ -602,6 +666,13 @@ function writePm2Config(dir, app, settings, env) {
602
666
  entry.env[LAUNCHER_COMMAND_ENV] = start.command;
603
667
  }
604
668
 
669
+ // Fork (the default) is left exactly as it was: PM2's own default. Cluster
670
+ // needs both fields, and a rollback reads them back from the release meta.
671
+ if (settings.execMode === 'cluster') {
672
+ entry.exec_mode = 'cluster';
673
+ entry.instances = settings.instances === 'max' ? 'max' : (settings.instances ?? 1);
674
+ }
675
+
605
676
  const file = path.join(dir, PM2_CONFIG_FILE);
606
677
  // Holds the app's environment, so owner-only like the deploy key.
607
678
  fs.writeFileSync(file, JSON.stringify({ apps: [entry] }, null, 2), { mode: 0o600 });
@@ -620,9 +691,14 @@ function releaseHealth(dir) {
620
691
  sha: typeof meta.sha === 'string' ? meta.sha : null,
621
692
  ref: typeof meta.ref === 'string' ? meta.ref : null,
622
693
  persist: Array.isArray(meta.persist) ? meta.persist.filter((p) => typeof p === 'string') : [],
694
+ execMode: meta.execMode === 'cluster' ? 'cluster' : 'fork',
695
+ instances: meta.instances === 'max' || Number.isInteger(meta.instances) ? meta.instances : 1,
623
696
  };
624
697
  } catch {
625
- return { port: null, healthPath: '/', strict: false, sha: null, ref: null, persist: [] };
698
+ return {
699
+ port: null, healthPath: '/', strict: false, sha: null, ref: null, persist: [],
700
+ execMode: 'fork', instances: 1,
701
+ };
626
702
  }
627
703
  }
628
704
 
@@ -635,70 +711,230 @@ export function describeRelease(appId, releaseId) {
635
711
  id: releaseId,
636
712
  sha: meta.sha,
637
713
  ref: meta.ref,
714
+ execMode: meta.execMode,
715
+ instances: meta.instances,
638
716
  startable: fs.existsSync(path.join(dir, PM2_CONFIG_FILE)),
639
717
  };
640
718
  }
641
719
 
720
+ /**
721
+ * Reads one app's PM2 status through core/runner.js, as a list of instances.
722
+ *
723
+ * A cluster app has one PM2 entry per worker, so statusAll is preferred. The
724
+ * older `status` and `list` fallbacks keep injected runners working; they
725
+ * report one entry, which is all a fork app has anyway.
726
+ *
727
+ * @returns {Promise<{ read: boolean, apps: Object[] }>}
728
+ */
729
+ async function readPm2Status(name, pm2Path, d) {
730
+ if (typeof d.runner.statusAll === 'function') {
731
+ const r = await d.runner.statusAll(name, { run: d.run, pm2Path });
732
+ if (!r?.ok) return { read: false, apps: [] };
733
+ return { read: true, apps: Array.isArray(r.apps) ? r.apps : [] };
734
+ }
735
+ if (typeof d.runner.status === 'function') {
736
+ const r = await d.runner.status(name, { run: d.run, pm2Path });
737
+ if (!r?.ok) return { read: false, apps: [] };
738
+ return { read: true, apps: r.app ? [r.app] : [] };
739
+ }
740
+ const listed = await d.runner.list({ run: d.run, pm2Path });
741
+ if (!listed.ok) return { read: false, apps: [] };
742
+ return { read: true, apps: listed.apps.filter((a) => a.name === name) };
743
+ }
744
+
745
+ /**
746
+ * Turns one PM2 observation into a boot-crash reason, or null when it is fine.
747
+ *
748
+ * `baseline` maps each instance to the restart count on the first observation
749
+ * after the start. Anything above its baseline means that worker exited at
750
+ * least once; PM2's state `errored`/`stopped` means it has given up. Both are
751
+ * boot crashes even when an HTTP probe happened to answer once, which is the
752
+ * case the health check alone cannot see. A cluster app is only healthy when
753
+ * EVERY worker is.
754
+ *
755
+ * @param {{ read: boolean, apps: Object[] }} observed
756
+ * @param {Map<number|string, number>|null} baseline
757
+ * @param {{ port?: number|null }} ctx
758
+ * @returns {string|null}
759
+ */
760
+ function bootCrashReason(observed, baseline, { port = null } = {}) {
761
+ if (!observed.read) return null;
762
+ const apps = Array.isArray(observed.apps) ? observed.apps : [];
763
+ if (apps.length === 0) return 'crashed at boot: PM2 no longer lists the process';
764
+
765
+ for (let i = 0; i < apps.length; i++) {
766
+ const app = apps[i];
767
+ const key = app.pmId ?? i;
768
+ const restarts = Number.isFinite(app.restarts) ? app.restarts : 0;
769
+
770
+ if (baseline && baseline.has(key) && restarts > baseline.get(key)) {
771
+ return `crashed at boot: restarting ${restarts} times`;
772
+ }
773
+ if (app.status === 'errored' || app.status === 'stopped') {
774
+ const where = port ? ` instead of listening on port ${port}` : '';
775
+ return `crashed at boot: process is "${app.status}" — it exited${where}`;
776
+ }
777
+ }
778
+
779
+ return null;
780
+ }
781
+
642
782
  /**
643
783
  * Waits for a freshly started app to be healthy.
644
- * @returns {Promise<{ ok: boolean, detail: string }>}
784
+ *
785
+ * Healthy means both: the HTTP check (or the no-port settle) passes, AND PM2
786
+ * keeps it online without increasing its restart count over a short
787
+ * confirmation window. The second half is what catches a release that answers
788
+ * a probe and then crashes at boot.
789
+ *
790
+ * @returns {Promise<{ ok: boolean, detail: string, crash: boolean, reason: string|null }>}
645
791
  */
646
792
  async function waitHealthy({ name, port, healthPath, strict, timeoutS, pm2Path, d }) {
647
- const status = async () => {
648
- const listed = await d.runner.list({ run: d.run, pm2Path });
649
- if (!listed.ok) return null;
650
- return listed.apps.find((a) => a.name === name) ?? null;
793
+ let baseline = null;
794
+ let last = 'no response yet';
795
+
796
+ /** @returns {string|null} A boot-crash reason, or null */
797
+ const observe = (observed) => {
798
+ const reason = bootCrashReason(observed, baseline, { port });
799
+ if (observed.read && observed.apps.length > 0 && baseline === null) {
800
+ baseline = new Map();
801
+ observed.apps.forEach((app, i) => {
802
+ baseline.set(app.pmId ?? i, Number.isFinite(app.restarts) ? app.restarts : 0);
803
+ });
804
+ }
805
+ return reason;
651
806
  };
652
807
 
808
+ const failOrNull = (reason) => (reason
809
+ ? { ok: false, crash: true, detail: reason, reason }
810
+ : null);
811
+
653
812
  if (!port) {
654
813
  // Nothing to ask over HTTP: the app has to stay online, without PM2
655
814
  // restarting it, for a few seconds.
656
815
  await d.sleep(Math.min(2000, NO_PORT_SETTLE_MS));
657
- const early = await status();
816
+ const early = await readPm2Status(name, pm2Path, d);
817
+ const earlyCrash = failOrNull(observe(early));
818
+ if (earlyCrash) return earlyCrash;
819
+
658
820
  await d.sleep(NO_PORT_SETTLE_MS - 2000);
659
- const late = await status();
660
- if (!late) return { ok: false, detail: 'PM2 no longer lists the process.' };
661
- if (late.status !== 'online') return { ok: false, detail: `The process is "${late.status}", not online.` };
662
- if (early && late.restarts > early.restarts) {
663
- return { ok: false, detail: 'The process keeps restarting — it starts and then exits.' };
821
+ const late = await readPm2Status(name, pm2Path, d);
822
+ const lateCrash = failOrNull(observe(late));
823
+ if (lateCrash) return lateCrash;
824
+
825
+ if (late.apps.length === 0) return { ok: false, crash: true, detail: 'crashed at boot: PM2 no longer lists the process', reason: 'crashed at boot: PM2 no longer lists the process' };
826
+ const notOnline = late.apps.find((a) => a.status !== 'online');
827
+ if (notOnline) {
828
+ const reason = `crashed at boot: process is "${notOnline.status}", not online`;
829
+ return { ok: false, crash: true, detail: reason, reason };
664
830
  }
665
- return { ok: true, detail: 'The process stayed online (no port is configured, so no HTTP check was made).' };
831
+ return {
832
+ ok: true,
833
+ crash: false,
834
+ reason: null,
835
+ detail: 'The process stayed online (no port is configured, so no HTTP check was made).',
836
+ };
666
837
  }
667
838
 
668
839
  const deadline = d.now() + timeoutS * 1000;
669
- let last = 'no response yet';
840
+ let answeredOnce = false;
670
841
 
671
842
  for (;;) {
672
843
  const res = await d.probe(port, healthPath);
673
844
  if (res.ok && isHealthyStatus(res.status, strict)) {
674
- return { ok: true, detail: `GET ${healthPath} on port ${port} answered ${res.status}.` };
845
+ if (answeredOnce) {
846
+ const observed = await readPm2Status(name, pm2Path, d);
847
+ const crash = failOrNull(observe(observed));
848
+ if (crash) return crash;
849
+ // A PM2 status that could not be read is not evidence of a crash; the
850
+ // HTTP answer is what we have.
851
+ if (!observed.read || (observed.apps.length > 0 && observed.apps.every((a) => a.status === 'online'))) {
852
+ return { ok: true, crash: false, reason: null, detail: `GET ${healthPath} on port ${port} answered ${res.status}, and PM2 stayed online.` };
853
+ }
854
+ } else {
855
+ const first = await readPm2Status(name, pm2Path, d);
856
+ const crash = failOrNull(observe(first));
857
+ if (crash) return crash;
858
+ answeredOnce = true;
859
+ // Keep watching for one short window: an app that answers and then
860
+ // crash-loops must not be accepted on the strength of that first answer.
861
+ const remaining = Math.max(0, deadline - d.now());
862
+ await d.sleep(Math.min(BOOT_CONFIRM_MS, remaining));
863
+ continue;
864
+ }
675
865
  }
676
866
  last = res.ok ? `it answered ${res.status}` : `no answer (${res.error})`;
677
867
 
678
- // A process PM2 has given up on will never become healthy; stop waiting.
679
- const app = await status();
680
- if (app && (app.status === 'errored' || app.status === 'stopped')) {
681
- return { ok: false, detail: `The process is "${app.status}" — it exited instead of listening on port ${port}.` };
868
+ const observed = await readPm2Status(name, pm2Path, d);
869
+ const crash = failOrNull(observe(observed));
870
+ if (crash) return crash;
871
+
872
+ if (d.now() >= deadline) {
873
+ // A process that never reached "online" is a boot crash, not merely a
874
+ // slow health endpoint.
875
+ const notOnline = observed.read ? observed.apps.find((a) => a.status !== 'online') : null;
876
+ if (notOnline) {
877
+ const reason = `crashed at boot: process is "${notOnline.status}", not online`;
878
+ return { ok: false, crash: true, detail: reason, reason };
879
+ }
880
+ const detail = `No healthy response from GET ${healthPath} on port ${port} within ${timeoutS}s (${last}).`;
881
+ return { ok: false, crash: false, detail, reason: `health check failed: ${detail}` };
682
882
  }
683
-
684
- if (d.now() >= deadline) break;
685
883
  await d.sleep(HEALTH_POLL_MS);
686
884
  }
885
+ }
687
886
 
688
- return {
689
- ok: false,
690
- detail: `No healthy response from GET ${healthPath} on port ${port} within ${timeoutS}s (${last}).`,
691
- };
887
+ /**
888
+ * Reads the tail of a crashed release's PM2 out+error log, with the app's
889
+ * secret values removed.
890
+ *
891
+ * @returns {Promise<string[]>} At most BOOT_LOG_LINES lines
892
+ */
893
+ async function captureBootLog(name, app, pm2Path, d) {
894
+ if (typeof d.runner.logs !== 'function') return [];
895
+ try {
896
+ const read = await d.runner.logs(name, { lines: BOOT_LOG_LINES, run: d.run, pm2Path });
897
+ const text = Array.isArray(read?.lines) ? read.lines.join('\n') : String(read?.output ?? '');
898
+ return text
899
+ .replace(/\r\n/g, '\n')
900
+ .split('\n')
901
+ .filter((line) => line.trim() !== '')
902
+ .slice(-BOOT_LOG_LINES)
903
+ .map((line) => redactSecrets(line, app.env ?? {}, app.auth?.token ?? null));
904
+ } catch {
905
+ return [];
906
+ }
692
907
  }
693
908
 
694
- /** Stops whatever is running under the app's PM2 name and starts a release. */
695
- async function switchTo(dir, name, pm2Path, d, log) {
909
+ /**
910
+ * Starts a release under PM2, or reloads it when both the new release and the
911
+ * running app are in cluster mode.
912
+ *
913
+ * `delete` rather than `restart` is the ordinary path: the working directory
914
+ * and the environment belong to the release, and PM2 keeps the old ones across
915
+ * a restart. Deleting a name PM2 does not know fails harmlessly on a first
916
+ * deployment. Cluster-to-cluster is the exception — PM2 reload overlaps the
917
+ * workers, which is the zero-downtime handover; a first deployment or a
918
+ * fork↔cluster switch cannot overlap and takes the delete/start path.
919
+ *
920
+ * @param {{ execMode?: 'fork'|'cluster' }} [options]
921
+ */
922
+ async function switchTo(dir, name, pm2Path, d, log, { execMode = 'fork' } = {}) {
696
923
  const config = path.join(dir, PM2_CONFIG_FILE);
697
924
  if (!fs.existsSync(config)) return { ok: false, output: `${PM2_CONFIG_FILE} is missing from ${dir}.` };
698
925
 
699
- // `delete` rather than `restart`: the working directory and the environment
700
- // belong to the release, and PM2 keeps the old ones across a restart.
701
- // Deleting a name PM2 does not know fails harmlessly on a first deployment.
926
+ if (execMode === 'cluster' && typeof d.runner.reloadConfig === 'function') {
927
+ const current = await readPm2Status(name, pm2Path, d);
928
+ const running = current.read && current.apps.length > 0;
929
+ const allCluster = running && current.apps.every((a) => a.execMode === 'cluster');
930
+ if (allCluster) {
931
+ log.line(`$ pm2 reload ${PM2_CONFIG_FILE} (cluster mode, zero downtime)`);
932
+ const reloaded = await d.runner.reloadConfig(config, { run: d.run, pm2Path });
933
+ if (!reloaded.ok) log.line(reloaded.output || 'pm2 reload failed with no output');
934
+ return reloaded;
935
+ }
936
+ }
937
+
702
938
  await d.runner.del(name, { run: d.run, pm2Path });
703
939
  const started = await d.runner.startConfig(config, { run: d.run, pm2Path });
704
940
  if (!started.ok) log.line(started.output || 'pm2 start failed with no output');
@@ -706,7 +942,12 @@ async function switchTo(dir, name, pm2Path, d, log) {
706
942
  }
707
943
 
708
944
  async function notify(app, run, d) {
709
- const event = createDeployEvent(app.id, run.status, {
945
+ // A boot crash is a failed run, but when the previous release is serving
946
+ // again the notification should say so rather than only "failed".
947
+ const outcome = run.rollbackSucceeded === true && run.status === RUN_STATUS.FAILED
948
+ ? RUN_STATUS.ROLLED_BACK
949
+ : run.status;
950
+ const event = createDeployEvent(app.id, outcome, {
710
951
  sha: run.sha,
711
952
  reason: run.error,
712
953
  rollback: run.trigger === 'rollback',
@@ -766,6 +1007,12 @@ export function startDeployment(appId, { trigger = 'manual', ref = null, actor =
766
1007
  error: null,
767
1008
  warnings: [],
768
1009
  rolledBackTo: null,
1010
+ /** A short human reason for a boot crash or health failure, else null. */
1011
+ reason: null,
1012
+ /** Redacted PM2 out+error tail for a boot crash; empty otherwise. */
1013
+ logTail: [],
1014
+ /** true/false once an automatic rollback was attempted, null when none was. */
1015
+ rollbackSucceeded: null,
769
1016
  };
770
1017
  writeRun(run);
771
1018
 
@@ -811,18 +1058,34 @@ async function executeDeployment(app, run, d) {
811
1058
  };
812
1059
 
813
1060
  /** Runs the user's commands for one step, in the release directory. */
814
- const userCommands = async (step, title, commands, env) => {
1061
+ const userCommands = async (step, title, commands, env, { hint = null, includeOutput = false } = {}) => {
815
1062
  if (!commands || commands.length === 0) return;
816
1063
  setStep(step, title);
817
1064
  for (const command of commands) {
818
1065
  log.line(`$ ${command}`);
1066
+ let tail = '';
819
1067
  const r = await d.runCommandLine(command, {
820
- cwd: dir, env, timeout: STEP_TIMEOUT_MS, onData: (c) => log.output(c),
1068
+ cwd: dir,
1069
+ env,
1070
+ timeout: STEP_TIMEOUT_MS,
1071
+ onData: (c) => {
1072
+ log.output(c);
1073
+ if (includeOutput) {
1074
+ tail += String(c);
1075
+ if (tail.length > 4000) tail = tail.slice(-4000);
1076
+ }
1077
+ },
821
1078
  });
822
1079
  log.flush();
823
1080
  if (!r.success) {
824
1081
  const why = r.exitCode === null ? (r.stderr || 'it was stopped') : `exit code ${r.exitCode}`;
825
- throw new StepError(step, `${title} failed: "${command}" — ${why}`);
1082
+ const lines = includeOutput
1083
+ ? tail.replace(/\r\n/g, '\n').split('\n').map((l) => l.trim()).filter(Boolean).slice(-5)
1084
+ .map((l) => redactSecrets(l, app.env ?? {}, app.auth?.token ?? null))
1085
+ : [];
1086
+ const detail = `${title} failed: "${command}" — ${why}${lines.length ? `\n${lines.join('\n')}` : ''}${hint ? `\n${hint}` : ''}`;
1087
+ if (hint) log.line(`hint: ${hint}`);
1088
+ throw new StepError(step, detail);
826
1089
  }
827
1090
  }
828
1091
  };
@@ -866,10 +1129,16 @@ async function executeDeployment(app, run, d) {
866
1129
 
867
1130
  const { settings, sources } = project;
868
1131
  log.line(project.ezLocation ? `ez.json found at ${project.ezLocation}` : 'no ez.json — using package.json defaults');
869
- for (const field of ['install', 'build', 'start', 'port', 'healthPath']) {
1132
+ for (const field of ['install', 'build', 'migrate', 'start', 'runtime', 'port', 'healthPath', 'execMode', 'instances']) {
870
1133
  const value = Array.isArray(settings[field]) ? (settings[field].join(' && ') || '(none)') : (settings[field] ?? '(none)');
871
1134
  log.line(` ${field.padEnd(10)} ${value} [${sources[field]}]`);
872
1135
  }
1136
+ if (settings.steps.length > 0) {
1137
+ const shown = settings.steps
1138
+ .map((s, i) => `${i + 1}.${s.kind}${s.enabled === false ? ' (disabled)' : ''}: ${s.run}`)
1139
+ .join(' | ');
1140
+ log.line(` steps ${shown} [${sources.steps}]`);
1141
+ }
873
1142
  if (settings.persist.length > 0) {
874
1143
  log.line(` persist ${settings.persist.join(', ')} [${sources.persist}]`);
875
1144
  }
@@ -883,10 +1152,30 @@ async function executeDeployment(app, run, d) {
883
1152
  const env = appEnvironment(app, settings, run.releaseId);
884
1153
  const strict = sources.healthPath !== 'default';
885
1154
 
886
- // ── 3. pre → install → build ────────────────────────────────────────────
1155
+ // ── 3. pre → steps (install/build/migrate/custom) ───────────────────────
887
1156
  await userCommands('pre', 'Pre-deploy hook', settings.pre, env);
888
- await userCommands('install', 'Install', settings.install, env);
889
- await userCommands('build', 'Build', settings.build, env);
1157
+ for (let i = 0; i < settings.steps.length; i++) {
1158
+ const step = settings.steps[i];
1159
+ const number = i + 1;
1160
+ if (step.enabled === false) {
1161
+ log.line(`Skipping step ${number} (${step.kind}): disabled.`);
1162
+ continue;
1163
+ }
1164
+ // The run record's `step` is the kind for the three steps that have
1165
+ // special behaviour, and "step N" for a custom one.
1166
+ const stepId = step.kind === 'custom' ? `step ${number}` : step.kind;
1167
+ const title = step.kind === 'install'
1168
+ ? 'Install'
1169
+ : step.kind === 'build'
1170
+ ? 'Build'
1171
+ : step.kind === 'migrate'
1172
+ ? 'Migrate'
1173
+ : `Step ${number}`;
1174
+ await userCommands(stepId, title, [step.run], env, {
1175
+ hint: step.kind === 'migrate' ? MIGRATE_HINT : null,
1176
+ includeOutput: step.kind === 'migrate',
1177
+ });
1178
+ }
890
1179
  verifyPersistentLinks(app.id, dir, settings.persist);
891
1180
 
892
1181
  // ── 4. switch ───────────────────────────────────────────────────────────
@@ -902,12 +1191,14 @@ async function executeDeployment(app, run, d) {
902
1191
  fs.writeFileSync(path.join(dir, 'ez.release.json'), JSON.stringify({
903
1192
  sha: run.sha, ref: run.ref, port: settings.port, healthPath: settings.healthPath, strict,
904
1193
  persist: settings.persist,
1194
+ execMode: settings.execMode,
1195
+ instances: settings.instances,
905
1196
  deployedAt: new Date(d.now()).toISOString(),
906
1197
  }, null, 2));
907
1198
 
908
- log.line(`$ pm2 delete ${name}; pm2 start ${PM2_CONFIG_FILE}`);
1199
+ log.line(`$ pm2 delete ${name}; pm2 start ${PM2_CONFIG_FILE}${settings.execMode === 'cluster' ? ' (cluster mode; a running cluster app is reloaded instead)' : ''}`);
909
1200
  switched = true;
910
- const started = await switchTo(dir, name, pm2Path, d, log);
1201
+ const started = await switchTo(dir, name, pm2Path, d, log, { execMode: settings.execMode });
911
1202
  if (!started.ok) throw new StepError('start', 'PM2 could not start the new release.');
912
1203
 
913
1204
  // ── 5. health ───────────────────────────────────────────────────────────
@@ -917,7 +1208,9 @@ async function executeDeployment(app, run, d) {
917
1208
  timeoutS: settings.healthTimeout, pm2Path, d,
918
1209
  });
919
1210
  log.line(health.detail);
920
- if (!health.ok) throw new StepError('health', health.detail);
1211
+ if (!health.ok) {
1212
+ throw new StepError('health', health.detail, { bootCrash: health.crash, reason: health.reason });
1213
+ }
921
1214
 
922
1215
  // ── 6. it is live ───────────────────────────────────────────────────────
923
1216
  writeState(app.id, { current: run.releaseId, previous: before.current });
@@ -952,25 +1245,40 @@ async function executeDeployment(app, run, d) {
952
1245
  log.line('Deployment succeeded.');
953
1246
  } catch (err) {
954
1247
  log.flush();
1248
+ const bootCrash = err?.bootCrash === true;
955
1249
  run.error = err?.message ?? String(err);
956
1250
  run.step = err?.step ?? run.step;
1251
+ run.reason = err?.reason ?? null;
957
1252
  log.line(`ERROR: ${run.error}`);
958
1253
  run.status = RUN_STATUS.FAILED;
959
1254
 
1255
+ // Read the crashed release's own PM2 log BEFORE a rollback reuses the
1256
+ // same PM2 name — afterwards these lines belong to the restored release.
1257
+ if (bootCrash) {
1258
+ run.logTail = await captureBootLog(name, app, pm2Path, d);
1259
+ if (run.logTail.length > 0) {
1260
+ log.step(`PM2 log (last ${run.logTail.length} lines, secrets redacted)`);
1261
+ for (const line of run.logTail) log.line(line);
1262
+ } else {
1263
+ log.line('PM2 produced no log lines for the crashed release.');
1264
+ }
1265
+ }
1266
+
960
1267
  if (!switched) {
961
1268
  // Nothing that was running has been touched. The half-built directory
962
1269
  // is of no use to a rollback and can be large; the log is what explains
963
1270
  // the failure.
964
1271
  try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
1272
+ run.logTail = [];
965
1273
  log.line('The running release was not touched.');
966
1274
  } else if (before.current && app.autoRollback !== false) {
967
1275
  log.step(`Rolling back to ${before.current}`);
968
1276
  let restored = false;
969
1277
  try {
970
1278
  const previousDir = releaseDir(app.id, before.current);
971
- const back = await switchTo(previousDir, name, pm2Path, d, log);
1279
+ const meta = releaseHealth(previousDir);
1280
+ const back = await switchTo(previousDir, name, pm2Path, d, log, { execMode: meta.execMode });
972
1281
  if (back.ok) {
973
- const meta = releaseHealth(previousDir);
974
1282
  const health = await waitHealthy({
975
1283
  name, port: meta.port, healthPath: meta.healthPath, strict: meta.strict,
976
1284
  timeoutS: ROLLBACK_HEALTH_TIMEOUT_S, pm2Path, d,
@@ -982,8 +1290,13 @@ async function executeDeployment(app, run, d) {
982
1290
  log.line(`Rollback error: ${rollbackErr.message}`);
983
1291
  }
984
1292
 
1293
+ run.rollbackSucceeded = restored;
985
1294
  if (restored) {
986
- run.status = RUN_STATUS.ROLLED_BACK;
1295
+ // A boot crash stays a failed run — what was deployed never worked —
1296
+ // but the fact that the previous release is serving again is recorded
1297
+ // rather than hidden. A plain health-check failure keeps the older
1298
+ // `rolled_back` status.
1299
+ if (!bootCrash) run.status = RUN_STATUS.ROLLED_BACK;
987
1300
  run.rolledBackTo = before.current;
988
1301
  log.line(`The previous release (${before.current}) is running again.`);
989
1302
  } else {
@@ -993,10 +1306,11 @@ async function executeDeployment(app, run, d) {
993
1306
  } else {
994
1307
  // No release to return to (a first deployment) or rollback is switched
995
1308
  // off. Remove the broken process so PM2 does not restart it in a loop.
1309
+ run.rollbackSucceeded = null;
996
1310
  try { await d.runner.del(name, { run: d.run, pm2Path }); } catch { /* ignore */ }
997
1311
  log.line(before.current
998
- ? 'Automatic rollback is switched off for this app; the process was stopped.'
999
- : 'There is no earlier release to roll back to; the process was stopped.');
1312
+ ? 'Automatic rollback is switched off for this app; the process was stopped. Fix the failure and deploy again.'
1313
+ : 'There is no earlier release to roll back to, so the new release was stopped. Fix the failure and deploy again.');
1000
1314
  }
1001
1315
  }
1002
1316
 
@@ -1058,6 +1372,9 @@ export function startRollback(appId, { releaseId = null, actor = null, deps = {}
1058
1372
  error: null,
1059
1373
  warnings: [],
1060
1374
  rolledBackTo: null,
1375
+ reason: null,
1376
+ logTail: [],
1377
+ rollbackSucceeded: null,
1061
1378
  };
1062
1379
  writeRun(run);
1063
1380
 
@@ -1073,7 +1390,7 @@ export function startRollback(appId, { releaseId = null, actor = null, deps = {}
1073
1390
  pm2Path = pm2.path ?? 'pm2';
1074
1391
 
1075
1392
  log.step(`Starting release ${target}`);
1076
- const started = await switchTo(targetDir, name, pm2Path, d, log);
1393
+ const started = await switchTo(targetDir, name, pm2Path, d, log, { execMode: meta.execMode });
1077
1394
  if (!started.ok) throw new StepError('start', 'PM2 could not start that release.');
1078
1395
 
1079
1396
  run.step = 'health';
@@ -1084,7 +1401,7 @@ export function startRollback(appId, { releaseId = null, actor = null, deps = {}
1084
1401
  timeoutS: ROLLBACK_HEALTH_TIMEOUT_S, pm2Path, d,
1085
1402
  });
1086
1403
  log.line(health.detail);
1087
- if (!health.ok) throw new StepError('health', health.detail);
1404
+ if (!health.ok) throw new StepError('health', health.detail, { bootCrash: health.crash, reason: health.reason });
1088
1405
 
1089
1406
  writeState(appId, { current: target, previous: before.current });
1090
1407
  await d.runner.save({ run: d.run, pm2Path });
@@ -1094,13 +1411,15 @@ export function startRollback(appId, { releaseId = null, actor = null, deps = {}
1094
1411
  } catch (err) {
1095
1412
  run.status = RUN_STATUS.FAILED;
1096
1413
  run.error = err?.message ?? String(err);
1414
+ run.reason = err?.reason ?? null;
1097
1415
  run.step = err?.step ?? run.step;
1098
1416
  log.line(`ERROR: ${run.error}`);
1099
1417
 
1100
1418
  // Put back what was running, so a failed rollback is not also an outage.
1101
1419
  if (before.current) {
1102
1420
  try {
1103
- const back = await switchTo(releaseDir(appId, before.current), name, pm2Path, d, log);
1421
+ const restoreDir = releaseDir(appId, before.current);
1422
+ const back = await switchTo(restoreDir, name, pm2Path, d, log, { execMode: releaseHealth(restoreDir).execMode });
1104
1423
  log.line(back.ok
1105
1424
  ? `Release ${before.current} was started again.`
1106
1425
  : `Release ${before.current} could NOT be started again. The app may be down.`);