easy-devops 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -6
- package/dist/ui/assets/index-B6zdwEnG.css +1 -0
- package/dist/ui/assets/index-Cy0oNmul.js +442 -0
- package/dist/ui/assets/socket-DwLPrgsr.js +1 -0
- package/dist/ui/assets/socket-K5ttLD2q.js +65 -0
- package/dist/ui/brand/apple-touch-icon.png +0 -0
- package/dist/ui/brand/favicon-16.png +0 -0
- package/dist/ui/brand/favicon-32.png +0 -0
- package/dist/ui/brand/favicon-light-16.png +0 -0
- package/dist/ui/brand/favicon-light-32.png +0 -0
- package/dist/ui/brand/favicon.ico +0 -0
- package/dist/ui/brand/icon-192.png +0 -0
- package/dist/ui/brand/icon-48.png +0 -0
- package/dist/ui/brand/icon-512.png +0 -0
- package/dist/ui/brand/icon-64-light.png +0 -0
- package/dist/ui/brand/icon-64.png +0 -0
- package/dist/ui/brand/logo-dark.png +0 -0
- package/dist/ui/brand/logo-light.png +0 -0
- package/dist/ui/brand/mark-dark.png +0 -0
- package/dist/ui/brand/mark-light.png +0 -0
- package/dist/ui/index.html +71 -0
- package/install.ps1 +188 -1
- package/install.sh +254 -29
- package/package.json +15 -4
- package/src/cli/dash.js +110 -0
- package/src/cli/index.js +237 -106
- package/src/cli/managers/domain-manager.js +60 -40
- package/src/cli/managers/ssl-manager.js +47 -330
- package/src/cli/menu.js +156 -0
- package/src/cli/menus/cleanup.js +205 -0
- package/src/cli/menus/dashboard.js +92 -1
- package/src/cli/menus/runtime.js +236 -0
- package/src/cli/menus/service.js +764 -0
- package/src/cli/menus/settings.js +176 -8
- package/src/cli/menus/uninstall.js +167 -0
- package/src/cli/menus/update.js +221 -65
- package/src/core/acme-challenge.js +273 -0
- package/src/core/cert-operations.js +189 -0
- package/src/{dashboard/lib → core}/cert-reader.js +2 -2
- package/src/core/cert-renewal.js +494 -0
- package/src/core/cleanup.js +600 -0
- package/src/core/config.js +169 -2
- package/src/core/daemon/ipc.js +292 -0
- package/src/core/daemon/lock.js +294 -0
- package/src/core/daemon/run.js +241 -0
- package/src/core/daemon/stopRequest.js +162 -0
- package/src/core/db.js +37 -8
- package/src/core/deploy/apps.js +491 -0
- package/src/core/deploy/domainLink.js +200 -0
- package/src/core/deploy/engine.js +973 -0
- package/src/core/deploy/ezfile.js +361 -0
- package/src/core/deploy/github.js +239 -0
- package/src/core/deploy/githubApi.js +261 -0
- package/src/core/deploy/hookServer.js +325 -0
- package/src/core/deploy/hubCommands.js +267 -0
- package/src/core/deploy/launcher.js +57 -0
- package/src/core/deploy/status.js +91 -0
- package/src/core/deploy/store.js +398 -0
- package/src/core/domainNotifier.js +94 -28
- package/src/core/domains-db.js +234 -0
- package/src/core/events.js +94 -2
- package/src/core/hub/agent.js +842 -0
- package/src/core/hubLink.js +306 -0
- package/src/core/hubPermissions.js +430 -0
- package/src/core/installSource.js +162 -0
- package/src/core/jobs/bus.js +120 -0
- package/src/core/jobs/index.js +371 -0
- package/src/core/jobs/monitors.js +368 -0
- package/src/core/jobs/owner.js +217 -0
- package/src/core/jobs/revisionWatcher.js +106 -0
- package/src/core/nginx-apply.js +213 -0
- package/src/core/nginx-conf-generator.js +253 -17
- package/src/{dashboard/lib → core}/nginx-service.js +18 -9
- package/src/core/notifications/claim.js +214 -0
- package/src/core/notifications/dispatch.js +154 -0
- package/src/core/notifier.js +31 -1
- package/src/core/passwords.js +172 -0
- package/src/core/permissions.js +8 -4
- package/src/core/platform.js +709 -2
- package/src/core/probeCache.js +258 -0
- package/src/core/purge.js +372 -0
- package/src/core/runner.js +488 -0
- package/src/core/service/acl.js +226 -0
- package/src/core/service/dataDir.js +345 -0
- package/src/core/service/index.js +1128 -0
- package/src/core/service/legacy.js +218 -0
- package/src/core/service/runtime.js +446 -0
- package/src/core/service/runtimeUpdate.js +800 -0
- package/src/core/service/systemd.js +458 -0
- package/src/core/service/windowsPersonal.js +530 -0
- package/src/core/service/windowsService.js +576 -0
- package/src/core/service/winsw.js +260 -0
- package/src/core/setupCode.js +331 -0
- package/src/core/shell.js +70 -7
- package/src/core/ssl-issuer.js +627 -0
- package/src/core/updater.js +295 -0
- package/src/core/validators.js +99 -0
- package/src/dashboard/controllers/authController.js +302 -15
- package/src/dashboard/controllers/deploymentsController.js +479 -0
- package/src/dashboard/controllers/domainsController.js +39 -52
- package/src/dashboard/controllers/hubController.js +135 -0
- package/src/dashboard/controllers/nginxController.js +21 -3
- package/src/dashboard/controllers/processesController.js +368 -0
- package/src/dashboard/controllers/settingsController.js +180 -12
- package/src/dashboard/controllers/sslController.js +171 -10
- package/src/dashboard/lib/daemonBridge.js +163 -0
- package/src/dashboard/lib/localRequest.js +235 -0
- package/src/dashboard/lib/loginRateLimit.js +119 -0
- package/src/dashboard/lib/requireAuth.js +80 -0
- package/src/dashboard/public/js/app.js +489 -19
- package/src/dashboard/routes/auth.js +10 -10
- package/src/dashboard/routes/deployments.js +46 -0
- package/src/dashboard/routes/domains.js +9 -0
- package/src/dashboard/routes/hub.js +22 -0
- package/src/dashboard/routes/nginx.js +9 -0
- package/src/dashboard/routes/processes.js +29 -0
- package/src/dashboard/routes/settings.js +9 -0
- package/src/dashboard/routes/ssl.js +9 -0
- package/src/dashboard/server.js +288 -21
- package/src/dashboard/socketManager.js +171 -207
- package/src/dashboard/views/index.ejs +1 -0
- package/src/dashboard/views/partials/login.ejs +68 -7
- package/src/dashboard/views/partials/processes-panel.ejs +261 -0
- package/src/dashboard/views/partials/settings-panel.ejs +287 -0
- package/src/dashboard/views/partials/sidebar.ejs +2 -0
- package/src/dashboard/lib/domains-db.js +0 -51
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* core/jobs/bus.js
|
|
3
|
+
*
|
|
4
|
+
* The in-process channel between whatever is running the background jobs and
|
|
5
|
+
* whatever wants to show their output.
|
|
6
|
+
*
|
|
7
|
+
* Before this existed, the monitors WERE the socket layer: they polled nginx
|
|
8
|
+
* and called `target.emit(...)` directly, so they could only run inside a
|
|
9
|
+
* process that had a Socket.io server. That is precisely what stopped the work
|
|
10
|
+
* from moving into a daemon.
|
|
11
|
+
*
|
|
12
|
+
* The monitors now publish here and know nothing about transports. Two
|
|
13
|
+
* subscribers exist: the dashboard (forwards straight to connected browsers)
|
|
14
|
+
* and the daemon's IPC server (forwards to a dashboard in another process).
|
|
15
|
+
*
|
|
16
|
+
* ── Why the event names are the browser's names ────────────────────────────
|
|
17
|
+
*
|
|
18
|
+
* `nginx:status`, `domain:health`, `notification:new` and `local:changed` are
|
|
19
|
+
* the names in docs/api-contract.md, and they are used unchanged here. A
|
|
20
|
+
* translation table between internal and external names would be one more
|
|
21
|
+
* thing to keep in step with the contract, and the contract is the thing that
|
|
22
|
+
* must not drift. So a forwarder is a one-liner: subscribe, emit the same name
|
|
23
|
+
* with the same payload.
|
|
24
|
+
*
|
|
25
|
+
* `logs:line` is deliberately NOT here. A log tail exists only for a client
|
|
26
|
+
* that is watching, so it belongs to the socket layer and dies with the socket.
|
|
27
|
+
*
|
|
28
|
+
* `service:changed` is published only by the DASHBOARD (dashboard/lib/
|
|
29
|
+
* daemonBridge.js): it describes the background service as that process sees
|
|
30
|
+
* it, including whether this process is the one running the jobs. It is listed
|
|
31
|
+
* here so the forwarding and replay machinery treats it like any other event;
|
|
32
|
+
* the daemon never publishes it, so nothing conflicting crosses the IPC link.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { EventEmitter } from 'events';
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Every event the background jobs publish.
|
|
39
|
+
*
|
|
40
|
+
* Exported as a frozen list so a forwarder can iterate it rather than repeat
|
|
41
|
+
* the names — a new event then reaches every subscriber without anyone having
|
|
42
|
+
* to remember to add it in three places.
|
|
43
|
+
*/
|
|
44
|
+
export const JOB_EVENTS = Object.freeze([
|
|
45
|
+
'nginx:status',
|
|
46
|
+
'domain:health',
|
|
47
|
+
'notification:new',
|
|
48
|
+
'local:changed',
|
|
49
|
+
'service:changed',
|
|
50
|
+
]);
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* One emitter per process. The jobs run in exactly one process at a time, so
|
|
54
|
+
* a module-level instance is the right scope: there is nothing to multiplex.
|
|
55
|
+
*/
|
|
56
|
+
const bus = new EventEmitter();
|
|
57
|
+
|
|
58
|
+
// The monitors, the hub agent and the IPC server can all subscribe, and a
|
|
59
|
+
// dashboard with several internal listeners is normal. The default limit of 10
|
|
60
|
+
// would start printing warnings that mean nothing here.
|
|
61
|
+
bus.setMaxListeners(50);
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Publishes a job event.
|
|
65
|
+
*
|
|
66
|
+
* Never throws: a listener that fails must not take down the monitor that
|
|
67
|
+
* produced the event, or one broken socket stops nginx from being watched.
|
|
68
|
+
*
|
|
69
|
+
* @param {string} event One of JOB_EVENTS
|
|
70
|
+
* @param {*} payload
|
|
71
|
+
*/
|
|
72
|
+
export function publish(event, payload) {
|
|
73
|
+
try {
|
|
74
|
+
bus.emit(event, payload);
|
|
75
|
+
} catch (err) {
|
|
76
|
+
console.error(`[jobs/bus] listener for "${event}" threw:`, err.message);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Subscribes to one event.
|
|
82
|
+
* @param {string} event
|
|
83
|
+
* @param {Function} listener
|
|
84
|
+
* @returns {() => void} Unsubscribe
|
|
85
|
+
*/
|
|
86
|
+
export function subscribe(event, listener) {
|
|
87
|
+
bus.on(event, listener);
|
|
88
|
+
return () => bus.off(event, listener);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Subscribes to every job event at once.
|
|
93
|
+
*
|
|
94
|
+
* This is what a transport wants: it forwards all of them identically, and
|
|
95
|
+
* using this means a newly added event is forwarded without a code change.
|
|
96
|
+
*
|
|
97
|
+
* @param {(event: string, payload: *) => void} listener
|
|
98
|
+
* @returns {() => void} Unsubscribe from all of them
|
|
99
|
+
*/
|
|
100
|
+
export function subscribeAll(listener) {
|
|
101
|
+
const offs = JOB_EVENTS.map((event) =>
|
|
102
|
+
subscribe(event, (payload) => listener(event, payload)),
|
|
103
|
+
);
|
|
104
|
+
return () => { for (const off of offs) off(); };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Removes every listener.
|
|
109
|
+
*
|
|
110
|
+
* For a dashboard that stops running the jobs and hands them to a daemon: its
|
|
111
|
+
* forwarders must go, or a later re-subscribe would double every event.
|
|
112
|
+
*/
|
|
113
|
+
export function clearAll() {
|
|
114
|
+
bus.removeAllListeners();
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Listener count, for diagnostics and tests. */
|
|
118
|
+
export function listenerCount(event) {
|
|
119
|
+
return bus.listenerCount(event);
|
|
120
|
+
}
|
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* core/jobs/index.js
|
|
3
|
+
*
|
|
4
|
+
* Decides WHICH process runs the background jobs, and hands them over when
|
|
5
|
+
* that answer changes.
|
|
6
|
+
*
|
|
7
|
+
* The rule, from the Phase 1 brief:
|
|
8
|
+
*
|
|
9
|
+
* The jobs run in exactly one process. The daemon runs them whenever it is
|
|
10
|
+
* running. An open dashboard runs them only while no daemon is running, and
|
|
11
|
+
* gives them up as soon as one appears.
|
|
12
|
+
*
|
|
13
|
+
* Two roles, one module, because the hand-over is a single piece of logic seen
|
|
14
|
+
* from both ends and splitting it would let the two halves disagree:
|
|
15
|
+
*
|
|
16
|
+
* - OWNER role ('daemon'): takes the lock, keeps the heartbeat alive, runs
|
|
17
|
+
* the jobs for as long as it holds it. If it ever loses the lock it stops
|
|
18
|
+
* the jobs rather than carrying on — a daemon that lost its lock is, by
|
|
19
|
+
* definition, no longer the one process.
|
|
20
|
+
*
|
|
21
|
+
* - STANDBY role ('dashboard'): never takes the lock. Polls it, runs the
|
|
22
|
+
* jobs while nobody holds it, and stops within one poll of a daemon
|
|
23
|
+
* appearing. On a fresh start it runs them immediately if no daemon is
|
|
24
|
+
* there, so a user with no service installed still gets monitoring.
|
|
25
|
+
*
|
|
26
|
+
* ── Why the dashboard does not take the lock ───────────────────────────────
|
|
27
|
+
*
|
|
28
|
+
* Tempting, and wrong: the daemon is the intended long-term owner, and a
|
|
29
|
+
* dashboard holding the lock would make `ezz daemon` refuse to start — a user
|
|
30
|
+
* installing the service would have to close their browser tab to get it
|
|
31
|
+
* running. The asymmetry is deliberate: the lock means "a daemon is running",
|
|
32
|
+
* not "someone is running the jobs".
|
|
33
|
+
*
|
|
34
|
+
* The cost is a window during hand-over where neither side is running them
|
|
35
|
+
* (dashboard stops, daemon starts) or, briefly, both are. Both are bounded by
|
|
36
|
+
* one poll interval, and the notification claim in core/notifications/claim.js
|
|
37
|
+
* is what makes the overlap harmless: even with two runners, a transition is
|
|
38
|
+
* sent exactly once.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
import {
|
|
42
|
+
acquire,
|
|
43
|
+
heartbeat,
|
|
44
|
+
release,
|
|
45
|
+
inspect,
|
|
46
|
+
isDaemonRunning,
|
|
47
|
+
watchLock,
|
|
48
|
+
HEARTBEAT_INTERVAL_MS,
|
|
49
|
+
} from '../daemon/lock.js';
|
|
50
|
+
import { sweepAbandoned } from '../notifications/claim.js';
|
|
51
|
+
import {
|
|
52
|
+
claim as claimOwner,
|
|
53
|
+
release as releaseOwner,
|
|
54
|
+
heartbeat as heartbeatOwner,
|
|
55
|
+
waitUntilFree,
|
|
56
|
+
HANDOVER_WAIT_MS,
|
|
57
|
+
OWNER_HEARTBEAT_MS,
|
|
58
|
+
} from './owner.js';
|
|
59
|
+
import { initHubAgent, stopHubAgent, reconcileHubAgent } from '../hub/agent.js';
|
|
60
|
+
import { startMonitors } from './monitors.js';
|
|
61
|
+
import { startDeployHookManager } from '../deploy/hookServer.js';
|
|
62
|
+
import { startRevisionWatcher } from './revisionWatcher.js';
|
|
63
|
+
|
|
64
|
+
/** How often a standby re-checks whether a daemon has appeared or gone. */
|
|
65
|
+
export const OWNERSHIP_POLL_MS = 15_000;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Asks the agent to match the stored pairing, when the revision counter moves.
|
|
69
|
+
*
|
|
70
|
+
* The decision lives in `core/hub/agent.js` because it needs the agent's own
|
|
71
|
+
* state: whether a socket exists and which pairing it was created for. An
|
|
72
|
+
* earlier version decided here from `getHubAgentStatus().connected`, and that
|
|
73
|
+
* was wrong — the counter moves on every domain, certificate and permission
|
|
74
|
+
* change, so a busy server redialled the socket constantly, including while it
|
|
75
|
+
* was still mid-connect. Hub updates then arrived seconds late.
|
|
76
|
+
*/
|
|
77
|
+
function onRevisionChange() {
|
|
78
|
+
try {
|
|
79
|
+
const result = reconcileHubAgent();
|
|
80
|
+
if (result.action !== 'none') {
|
|
81
|
+
console.log(`[jobs] hub agent ${result.action}: ${result.reason}`);
|
|
82
|
+
}
|
|
83
|
+
} catch (err) {
|
|
84
|
+
console.error('[jobs] could not reconcile the hub agent:', err.message);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Starts the jobs under an ownership policy.
|
|
90
|
+
*
|
|
91
|
+
* @param {Object} opts
|
|
92
|
+
* @param {'daemon'|'dashboard'} opts.role
|
|
93
|
+
* @param {string} [opts.dataDir]
|
|
94
|
+
* @param {string} [opts.version] Recorded in the lock, shown by `service status`
|
|
95
|
+
* @param {Object} [opts.intervals] Monitor interval overrides, for tests
|
|
96
|
+
* @param {boolean} [opts.hub] Run the Hub agent as part of the jobs (default true)
|
|
97
|
+
* @param {number} [opts.pollMs] Ownership re-check interval
|
|
98
|
+
* @param {(state: JobState) => void} [opts.onChange] Called whenever ownership changes
|
|
99
|
+
* @returns {{ stop: () => Promise<void>, state: () => JobState, poll: () => void }
|
|
100
|
+
* | { ok: false, reason: string, lock: Object|null }}
|
|
101
|
+
*
|
|
102
|
+
* @typedef {Object} JobState
|
|
103
|
+
* @property {'daemon'|'dashboard'} role
|
|
104
|
+
* @property {boolean} running Whether this process is running the jobs
|
|
105
|
+
* @property {string} reason Why, in a form a log line or a banner can use
|
|
106
|
+
* @property {boolean} daemonSeen Whether a daemon holds the lock
|
|
107
|
+
*/
|
|
108
|
+
export function startJobs({
|
|
109
|
+
role,
|
|
110
|
+
dataDir = undefined,
|
|
111
|
+
version = null,
|
|
112
|
+
intervals = {},
|
|
113
|
+
hub = true,
|
|
114
|
+
pollMs = OWNERSHIP_POLL_MS,
|
|
115
|
+
handoverWaitMs = HANDOVER_WAIT_MS,
|
|
116
|
+
onChange = null,
|
|
117
|
+
} = {}) {
|
|
118
|
+
if (role !== 'daemon' && role !== 'dashboard') {
|
|
119
|
+
throw new Error(`startJobs: role must be 'daemon' or 'dashboard', got ${JSON.stringify(role)}`);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const lockOpts = dataDir ? { dataDir } : {};
|
|
123
|
+
|
|
124
|
+
if (role === 'daemon') {
|
|
125
|
+
const got = acquire({ ...lockOpts, version, mode: 'daemon' });
|
|
126
|
+
if (!got.ok) {
|
|
127
|
+
// The caller decides what to print: `ezz daemon` says another daemon is
|
|
128
|
+
// already running, the service wrapper treats it as "nothing to do".
|
|
129
|
+
return { ok: false, reason: got.reason, lock: got.lock };
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** @type {{ monitors: Object|null, revisions: Object|null }} */
|
|
134
|
+
const handles = { monitors: null, revisions: null, ownerBeat: null, deployHook: null };
|
|
135
|
+
let running = false;
|
|
136
|
+
let reason = 'starting';
|
|
137
|
+
let stopped = false;
|
|
138
|
+
|
|
139
|
+
const state = () => ({
|
|
140
|
+
role,
|
|
141
|
+
running,
|
|
142
|
+
reason,
|
|
143
|
+
daemonSeen: role === 'daemon' ? true : isDaemonRunning(lockOpts),
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
const announce = () => {
|
|
147
|
+
if (!onChange) return;
|
|
148
|
+
try {
|
|
149
|
+
onChange(state());
|
|
150
|
+
} catch (err) {
|
|
151
|
+
console.error('[jobs] onChange listener threw:', err.message);
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
const startRunning = (why) => {
|
|
156
|
+
if (running || stopped) return;
|
|
157
|
+
|
|
158
|
+
// Exactly one process may run the jobs, and the daemon lock alone could
|
|
159
|
+
// not enforce that: the dashboard never takes it. So ownership has its own
|
|
160
|
+
// marker, and refusing to start without it is what makes the hand-over
|
|
161
|
+
// exclusive rather than merely quick.
|
|
162
|
+
const claimed = claimOwner({ ...lockOpts, role });
|
|
163
|
+
if (!claimed.ok) {
|
|
164
|
+
if (reason !== claimed.reason) { reason = claimed.reason; announce(); }
|
|
165
|
+
console.log(`[jobs] not starting the jobs: ${claimed.reason}`);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
if (claimed.recovered) {
|
|
169
|
+
console.log('[jobs] recovered the jobs marker from a process that is gone');
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// A crashed owner can leave claims behind. Clearing the ones whose owners
|
|
173
|
+
// are gone means the first poll can send immediately instead of waiting
|
|
174
|
+
// out the claim TTL — without it, a daemon restart during an outage would
|
|
175
|
+
// stay silent for two minutes.
|
|
176
|
+
try {
|
|
177
|
+
const swept = sweepAbandoned(lockOpts);
|
|
178
|
+
if (swept > 0) console.log(`[jobs] cleared ${swept} abandoned notification claim(s)`);
|
|
179
|
+
} catch { /* non-fatal */ }
|
|
180
|
+
|
|
181
|
+
handles.monitors = startMonitors({ intervals });
|
|
182
|
+
|
|
183
|
+
// The revision counter is also how a pairing change reaches this process.
|
|
184
|
+
// Without this, pairing from the dashboard while the daemon owns the jobs
|
|
185
|
+
// left the daemon with no agent until it was restarted — and tempted the
|
|
186
|
+
// dashboard into starting one itself, which is the two-socket flap.
|
|
187
|
+
handles.revisions = startRevisionWatcher({
|
|
188
|
+
onChange: hub ? onRevisionChange : null,
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
// The hub agent is background work like any other, so it follows ownership
|
|
192
|
+
// exactly. It must: two agents connected with the same credentials would
|
|
193
|
+
// both receive every command from the Hub and both execute it, so an
|
|
194
|
+
// "apply this config" would run twice. Tying it to the same election as
|
|
195
|
+
// the monitors is what makes that impossible — rather than two call sites
|
|
196
|
+
// each deciding for themselves whether to start one.
|
|
197
|
+
if (hub) {
|
|
198
|
+
try {
|
|
199
|
+
initHubAgent();
|
|
200
|
+
} catch (err) {
|
|
201
|
+
console.error('[jobs] hub agent failed to start:', err.message);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// The deploy hook follows ownership for the same reason the agent does:
|
|
206
|
+
// two processes cannot both bind its port, and a CI job must reach the one
|
|
207
|
+
// that is actually going to run the deployment. It listens on nothing
|
|
208
|
+
// unless `deployHook.enabled` is set — see core/deploy/hookServer.js.
|
|
209
|
+
try {
|
|
210
|
+
handles.deployHook = startDeployHookManager();
|
|
211
|
+
} catch (err) {
|
|
212
|
+
console.error('[jobs] deploy hook failed to start:', err.message);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// The marker has to be refreshed or it goes stale in 20 seconds and
|
|
216
|
+
// another process would be entitled to take the jobs from under us.
|
|
217
|
+
handles.ownerBeat = setInterval(() => {
|
|
218
|
+
if (stopped || !running) return;
|
|
219
|
+
if (!heartbeatOwner({ ...lockOpts, role })) {
|
|
220
|
+
// Someone else owns the jobs now. Standing down is the safe direction,
|
|
221
|
+
// for the same reason the daemon stands down when it loses its lock.
|
|
222
|
+
console.error('[jobs] lost the jobs marker - standing down');
|
|
223
|
+
stopRunning('another process holds the jobs');
|
|
224
|
+
}
|
|
225
|
+
}, OWNER_HEARTBEAT_MS);
|
|
226
|
+
|
|
227
|
+
running = true;
|
|
228
|
+
reason = why;
|
|
229
|
+
console.log(`[jobs] running the background jobs (${why})`);
|
|
230
|
+
announce();
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
const stopRunning = (why) => {
|
|
234
|
+
if (!running) {
|
|
235
|
+
// Keep the reason current even when nothing was running, so a banner
|
|
236
|
+
// reads "a daemon is running" rather than the stale startup reason.
|
|
237
|
+
if (reason !== why) { reason = why; announce(); }
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
handles.monitors?.stop();
|
|
242
|
+
handles.revisions?.stop();
|
|
243
|
+
if (handles.ownerBeat) clearInterval(handles.ownerBeat);
|
|
244
|
+
// Its port must be free before another process is told the jobs are
|
|
245
|
+
// available; the close is quick, and is not awaited because this path is
|
|
246
|
+
// synchronous — a successor that finds the port briefly busy retries.
|
|
247
|
+
handles.deployHook?.stop().catch(() => {});
|
|
248
|
+
handles.monitors = null;
|
|
249
|
+
handles.revisions = null;
|
|
250
|
+
handles.ownerBeat = null;
|
|
251
|
+
handles.deployHook = null;
|
|
252
|
+
|
|
253
|
+
if (hub) {
|
|
254
|
+
try {
|
|
255
|
+
stopHubAgent();
|
|
256
|
+
} catch (err) {
|
|
257
|
+
console.error('[jobs] hub agent failed to stop cleanly:', err.message);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// Released LAST, after the agent and the monitors are actually down. A
|
|
262
|
+
// daemon waiting on this marker must not be told the jobs are free while
|
|
263
|
+
// a Hub socket is still open here — that is the overlap the marker exists
|
|
264
|
+
// to prevent.
|
|
265
|
+
releaseOwner(lockOpts);
|
|
266
|
+
|
|
267
|
+
running = false;
|
|
268
|
+
reason = why;
|
|
269
|
+
console.log(`[jobs] stopped the background jobs (${why})`);
|
|
270
|
+
announce();
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
// ── Role: daemon ──────────────────────────────────────────────────────────
|
|
274
|
+
|
|
275
|
+
let beat = null;
|
|
276
|
+
let ready = Promise.resolve();
|
|
277
|
+
|
|
278
|
+
if (role === 'daemon') {
|
|
279
|
+
// The lock is ALREADY taken at this point (above), which is what the
|
|
280
|
+
// dashboard's watcher reacts to. Starting the jobs is deliberately
|
|
281
|
+
// deferred until the dashboard has actually released them: taking the lock
|
|
282
|
+
// tells it to stand down, and this waits until it has.
|
|
283
|
+
//
|
|
284
|
+
// The wait is bounded. If it expires the jobs start anyway — a server with
|
|
285
|
+
// no monitoring is worse than a few seconds of overlap, and the
|
|
286
|
+
// notification claim makes an overlap harmless.
|
|
287
|
+
ready = (async () => {
|
|
288
|
+
const free = await waitUntilFree({ ...lockOpts, timeoutMs: handoverWaitMs });
|
|
289
|
+
|
|
290
|
+
if (stopped) return;
|
|
291
|
+
|
|
292
|
+
if (!free.free) {
|
|
293
|
+
console.warn(
|
|
294
|
+
`[jobs] starting anyway after waiting ${Math.round(free.waitedMs / 1000)}s `
|
|
295
|
+
+ `for the jobs to be released - ${free.reason}`,
|
|
296
|
+
);
|
|
297
|
+
} else if (free.waitedMs > 200) {
|
|
298
|
+
console.log(`[jobs] the previous owner released the jobs after ${free.waitedMs}ms`);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
startRunning('this process is the daemon');
|
|
302
|
+
})();
|
|
303
|
+
|
|
304
|
+
beat = setInterval(() => {
|
|
305
|
+
if (stopped) return;
|
|
306
|
+
const stillMine = heartbeat({ ...lockOpts, version, mode: 'daemon' });
|
|
307
|
+
if (stillMine) return;
|
|
308
|
+
|
|
309
|
+
// Someone else took the lock — a stale-lock recovery that fired while we
|
|
310
|
+
// were paused, or a second daemon started by a service manager that
|
|
311
|
+
// thought we were dead. Standing down is the safe direction: the other
|
|
312
|
+
// process is the one the lock file names, and two daemons competing for
|
|
313
|
+
// the hub socket is worse than one.
|
|
314
|
+
console.error('[jobs] lost the daemon lock - standing down');
|
|
315
|
+
stopRunning('another process holds the daemon lock');
|
|
316
|
+
clearInterval(beat);
|
|
317
|
+
beat = null;
|
|
318
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// ── Role: dashboard (standby) ─────────────────────────────────────────────
|
|
322
|
+
|
|
323
|
+
const poll = () => {
|
|
324
|
+
if (stopped || role !== 'dashboard') return;
|
|
325
|
+
|
|
326
|
+
const daemon = inspect(lockOpts);
|
|
327
|
+
if (daemon.running) {
|
|
328
|
+
stopRunning(`a daemon is running (pid ${daemon.lock?.pid})`);
|
|
329
|
+
} else {
|
|
330
|
+
startRunning('no daemon is running, so this dashboard runs them');
|
|
331
|
+
}
|
|
332
|
+
};
|
|
333
|
+
|
|
334
|
+
let watcher = null;
|
|
335
|
+
let lockWatch = null;
|
|
336
|
+
|
|
337
|
+
if (role === 'dashboard') {
|
|
338
|
+
poll();
|
|
339
|
+
watcher = setInterval(poll, pollMs);
|
|
340
|
+
|
|
341
|
+
// The poll alone was too slow in the direction that matters. A daemon
|
|
342
|
+
// appearing had to wait up to a full interval to be noticed, and for that
|
|
343
|
+
// whole window both processes ran the jobs — including two Hub agents with
|
|
344
|
+
// the same serverId, which the Hub evicts in turn. Watching the lock makes
|
|
345
|
+
// standing down immediate; the interval stays as the fallback for a
|
|
346
|
+
// platform where the watch cannot be established.
|
|
347
|
+
lockWatch = watchLock({ ...lockOpts, onChange: poll });
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
return {
|
|
351
|
+
/**
|
|
352
|
+
* Resolves once this process has decided whether to run the jobs.
|
|
353
|
+
*
|
|
354
|
+
* For a daemon that means the hand-over wait is over and the jobs are
|
|
355
|
+
* started; `startDaemon()` awaits it so it never reports itself ready
|
|
356
|
+
* while the previous owner is still shutting down.
|
|
357
|
+
*/
|
|
358
|
+
ready: () => ready,
|
|
359
|
+
|
|
360
|
+
async stop() {
|
|
361
|
+
stopped = true;
|
|
362
|
+
if (beat) clearInterval(beat);
|
|
363
|
+
if (watcher) clearInterval(watcher);
|
|
364
|
+
lockWatch?.close();
|
|
365
|
+
stopRunning('shutting down');
|
|
366
|
+
if (role === 'daemon') release(lockOpts);
|
|
367
|
+
},
|
|
368
|
+
state,
|
|
369
|
+
poll,
|
|
370
|
+
};
|
|
371
|
+
}
|