easy-devops 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -6
- package/dist/ui/assets/index-CMRzkIKj.css +1 -0
- package/dist/ui/assets/index-DEouL3IK.js +452 -0
- package/dist/ui/assets/socket-DwLPrgsr.js +1 -0
- package/dist/ui/assets/socket-K5ttLD2q.js +65 -0
- package/dist/ui/brand/apple-touch-icon.png +0 -0
- package/dist/ui/brand/favicon-16.png +0 -0
- package/dist/ui/brand/favicon-32.png +0 -0
- package/dist/ui/brand/favicon-light-16.png +0 -0
- package/dist/ui/brand/favicon-light-32.png +0 -0
- package/dist/ui/brand/favicon.ico +0 -0
- package/dist/ui/brand/icon-192.png +0 -0
- package/dist/ui/brand/icon-48.png +0 -0
- package/dist/ui/brand/icon-512.png +0 -0
- package/dist/ui/brand/icon-64-light.png +0 -0
- package/dist/ui/brand/icon-64.png +0 -0
- package/dist/ui/brand/logo-dark.png +0 -0
- package/dist/ui/brand/logo-light.png +0 -0
- package/dist/ui/brand/mark-dark.png +0 -0
- package/dist/ui/brand/mark-light.png +0 -0
- package/dist/ui/index.html +71 -0
- package/install.ps1 +188 -1
- package/install.sh +254 -29
- package/package.json +15 -4
- package/src/cli/dash.js +110 -0
- package/src/cli/index.js +237 -106
- package/src/cli/managers/domain-manager.js +60 -40
- package/src/cli/managers/ssl-manager.js +47 -330
- package/src/cli/menu.js +156 -0
- package/src/cli/menus/cleanup.js +205 -0
- package/src/cli/menus/dashboard.js +92 -1
- package/src/cli/menus/runtime.js +236 -0
- package/src/cli/menus/service.js +764 -0
- package/src/cli/menus/settings.js +176 -8
- package/src/cli/menus/uninstall.js +167 -0
- package/src/cli/menus/update.js +222 -47
- package/src/core/acme-challenge.js +273 -0
- package/src/core/cert-operations.js +189 -0
- package/src/{dashboard/lib → core}/cert-reader.js +2 -2
- package/src/core/cert-renewal.js +494 -0
- package/src/core/cleanup.js +600 -0
- package/src/core/config.js +169 -2
- package/src/core/daemon/ipc.js +292 -0
- package/src/core/daemon/lock.js +294 -0
- package/src/core/daemon/run.js +241 -0
- package/src/core/daemon/stopRequest.js +162 -0
- package/src/core/db.js +37 -8
- package/src/core/deploy/apps.js +491 -0
- package/src/core/deploy/domainLink.js +200 -0
- package/src/core/deploy/engine.js +973 -0
- package/src/core/deploy/ezfile.js +361 -0
- package/src/core/deploy/github.js +239 -0
- package/src/core/deploy/githubApi.js +261 -0
- package/src/core/deploy/hookServer.js +325 -0
- package/src/core/deploy/hubCommands.js +267 -0
- package/src/core/deploy/launcher.js +57 -0
- package/src/core/deploy/status.js +91 -0
- package/src/core/deploy/store.js +398 -0
- package/src/core/domainNotifier.js +94 -28
- package/src/core/domains-db.js +234 -0
- package/src/core/events.js +94 -2
- package/src/core/hub/agent.js +842 -0
- package/src/core/hubLink.js +306 -0
- package/src/core/hubPermissions.js +430 -0
- package/src/core/installSource.js +162 -0
- package/src/core/jobs/bus.js +120 -0
- package/src/core/jobs/index.js +371 -0
- package/src/core/jobs/monitors.js +368 -0
- package/src/core/jobs/owner.js +217 -0
- package/src/core/jobs/revisionWatcher.js +106 -0
- package/src/core/nginx-apply.js +213 -0
- package/src/core/nginx-conf-generator.js +253 -17
- package/src/{dashboard/lib → core}/nginx-service.js +18 -9
- package/src/core/notifications/claim.js +214 -0
- package/src/core/notifications/dispatch.js +154 -0
- package/src/core/notifier.js +31 -1
- package/src/core/passwords.js +172 -0
- package/src/core/permissions.js +8 -4
- package/src/core/platform.js +709 -2
- package/src/core/probeCache.js +258 -0
- package/src/core/purge.js +372 -0
- package/src/core/runner.js +488 -0
- package/src/core/service/acl.js +226 -0
- package/src/core/service/dataDir.js +345 -0
- package/src/core/service/index.js +1128 -0
- package/src/core/service/legacy.js +218 -0
- package/src/core/service/runtime.js +446 -0
- package/src/core/service/runtimeUpdate.js +800 -0
- package/src/core/service/systemd.js +458 -0
- package/src/core/service/windowsPersonal.js +530 -0
- package/src/core/service/windowsService.js +576 -0
- package/src/core/service/winsw.js +260 -0
- package/src/core/setupCode.js +331 -0
- package/src/core/shell.js +70 -7
- package/src/core/ssl-issuer.js +627 -0
- package/src/core/updater.js +295 -0
- package/src/core/validators.js +99 -0
- package/src/dashboard/controllers/authController.js +302 -15
- package/src/dashboard/controllers/deploymentsController.js +479 -0
- package/src/dashboard/controllers/domainsController.js +80 -61
- package/src/dashboard/controllers/hubController.js +135 -0
- package/src/dashboard/controllers/nginxController.js +21 -3
- package/src/dashboard/controllers/processesController.js +368 -0
- package/src/dashboard/controllers/settingsController.js +180 -12
- package/src/dashboard/controllers/sslController.js +288 -10
- package/src/dashboard/lib/daemonBridge.js +163 -0
- package/src/dashboard/lib/localRequest.js +235 -0
- package/src/dashboard/lib/loginRateLimit.js +119 -0
- package/src/dashboard/lib/requireAuth.js +80 -0
- package/src/dashboard/public/js/app.js +489 -19
- package/src/dashboard/routes/auth.js +10 -10
- package/src/dashboard/routes/deployments.js +46 -0
- package/src/dashboard/routes/domains.js +9 -0
- package/src/dashboard/routes/hub.js +22 -0
- package/src/dashboard/routes/nginx.js +9 -0
- package/src/dashboard/routes/processes.js +29 -0
- package/src/dashboard/routes/settings.js +9 -0
- package/src/dashboard/routes/ssl.js +10 -0
- package/src/dashboard/server.js +288 -21
- package/src/dashboard/socketManager.js +171 -207
- package/src/dashboard/views/index.ejs +1 -0
- package/src/dashboard/views/partials/login.ejs +68 -7
- package/src/dashboard/views/partials/processes-panel.ejs +261 -0
- package/src/dashboard/views/partials/settings-panel.ejs +287 -0
- package/src/dashboard/views/partials/sidebar.ejs +2 -0
- package/src/dashboard/lib/domains-db.js +0 -51
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* core/jobs/monitors.js
|
|
3
|
+
*
|
|
4
|
+
* The background monitoring that used to live inside the dashboard's socket
|
|
5
|
+
* layer: nginx status, per-domain backend health, certificate expiry — and,
|
|
6
|
+
* since it has to run where no browser is open, automatic certificate renewal.
|
|
7
|
+
*
|
|
8
|
+
* Moved here so it can run in the daemon with no HTTP server anywhere near it.
|
|
9
|
+
* The behaviour is the same, with two deliberate changes:
|
|
10
|
+
*
|
|
11
|
+
* 1. Results are published to core/jobs/bus.js instead of being emitted at a
|
|
12
|
+
* socket. Nothing here knows what a socket is.
|
|
13
|
+
* 2. Notifications go through core/notifications/dispatch.js, so a transition
|
|
14
|
+
* is claimed before it is sent and only recorded once it has been. The old
|
|
15
|
+
* code recorded it first, which made a failed send permanently invisible.
|
|
16
|
+
*
|
|
17
|
+
* Each monitor is a plain function plus an interval. `startMonitors()` returns
|
|
18
|
+
* a stop handle; nothing is global except the bus.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import http from 'http';
|
|
22
|
+
|
|
23
|
+
import { getStatus } from '../nginx-service.js';
|
|
24
|
+
import { listAllCerts } from '../cert-reader.js';
|
|
25
|
+
import { dbGet } from '../db.js';
|
|
26
|
+
import { loadConfig } from '../config.js';
|
|
27
|
+
import { getDomains } from '../domains-db.js';
|
|
28
|
+
import {
|
|
29
|
+
runAutoRenewal,
|
|
30
|
+
describeCertificate,
|
|
31
|
+
getAutoRenewSettings,
|
|
32
|
+
autoRenewPlan,
|
|
33
|
+
} from '../cert-renewal.js';
|
|
34
|
+
import {
|
|
35
|
+
createNginxDownEvent,
|
|
36
|
+
createNginxRecoveredEvent,
|
|
37
|
+
createDomainDownEvent,
|
|
38
|
+
createDomainRecoveredEvent,
|
|
39
|
+
createCertExpiryEvent,
|
|
40
|
+
} from '../events.js';
|
|
41
|
+
import { getEffectiveChannelIds } from '../domainNotifier.js';
|
|
42
|
+
import { sendNotification } from '../notifier.js';
|
|
43
|
+
import { dispatchTransition } from '../notifications/dispatch.js';
|
|
44
|
+
import { publish } from './bus.js';
|
|
45
|
+
|
|
46
|
+
/** Poll intervals, unchanged from the socket implementation. */
|
|
47
|
+
export const NGINX_POLL_MS = 5_000;
|
|
48
|
+
export const DOMAIN_HEALTH_POLL_MS = 60_000;
|
|
49
|
+
export const CERT_CHECK_MS = 12 * 60 * 60 * 1000;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* A certificate is worth warning about below this many days.
|
|
53
|
+
* The danger-vs-warning threshold is not here: createCertExpiryEvent() in
|
|
54
|
+
* core/events.js owns the severity, and two places deciding it would drift.
|
|
55
|
+
*/
|
|
56
|
+
export const CERT_WARN_DAYS = 30;
|
|
57
|
+
|
|
58
|
+
/** Backend health probe timeout. */
|
|
59
|
+
const HEALTH_TIMEOUT_MS = 5_000;
|
|
60
|
+
|
|
61
|
+
// ─── nginx status ─────────────────────────────────────────────────────────────
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Polls nginx and publishes its status, notifying on an up↔down transition.
|
|
65
|
+
*
|
|
66
|
+
* @returns {Promise<Object>} The status that was published
|
|
67
|
+
*/
|
|
68
|
+
export async function checkNginx() {
|
|
69
|
+
let status;
|
|
70
|
+
try {
|
|
71
|
+
status = await getStatus();
|
|
72
|
+
} catch (err) {
|
|
73
|
+
// A failed status read is itself a status: publish it so the UI shows the
|
|
74
|
+
// error rather than the last good value. It is NOT treated as "down" for
|
|
75
|
+
// notification purposes — we do not know that nginx is down, only that we
|
|
76
|
+
// could not ask, and crying wolf on a transient read error is worse than
|
|
77
|
+
// staying quiet.
|
|
78
|
+
status = { running: false, version: null, pid: null, error: err.message };
|
|
79
|
+
publish('nginx:status', status);
|
|
80
|
+
return status;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
publish('nginx:status', status);
|
|
84
|
+
|
|
85
|
+
await dispatchTransition({
|
|
86
|
+
domain: null,
|
|
87
|
+
eventType: 'nginx_down',
|
|
88
|
+
status: status.running ? 'up' : 'down',
|
|
89
|
+
buildEvent: (d) => (d.reason === 'recovery' ? createNginxRecoveredEvent() : createNginxDownEvent()),
|
|
90
|
+
emitLocal: (event) => publish('notification:new', event),
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
return status;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// ─── domain health ────────────────────────────────────────────────────────────
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* HEAD request against a domain's backend.
|
|
100
|
+
*
|
|
101
|
+
* @param {string} host
|
|
102
|
+
* @param {number|string} port
|
|
103
|
+
* @returns {Promise<{ up: boolean, status?: number }>}
|
|
104
|
+
*/
|
|
105
|
+
export function probeBackend(host, port) {
|
|
106
|
+
return new Promise((resolve) => {
|
|
107
|
+
const req = http.request(
|
|
108
|
+
{ host, port: Number(port), path: '/', method: 'HEAD', timeout: HEALTH_TIMEOUT_MS },
|
|
109
|
+
(res) => {
|
|
110
|
+
resolve({ up: true, status: res.statusCode });
|
|
111
|
+
res.resume();
|
|
112
|
+
},
|
|
113
|
+
);
|
|
114
|
+
req.on('timeout', () => { req.destroy(); resolve({ up: false }); });
|
|
115
|
+
req.on('error', () => resolve({ up: false }));
|
|
116
|
+
req.end();
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Probes a list of domains and returns the results. Pure measurement: nothing
|
|
122
|
+
* is published and nobody is notified, so a caller that only wants the answer
|
|
123
|
+
* (the Hub agent, before the first scheduled sweep) can ask without side
|
|
124
|
+
* effects.
|
|
125
|
+
*
|
|
126
|
+
* @param {Array<Object>} domains
|
|
127
|
+
* @returns {Promise<Array<{ name: string, up: boolean|null, port?: number, host?: string }>>}
|
|
128
|
+
*/
|
|
129
|
+
export function probeDomains(domains) {
|
|
130
|
+
return Promise.all(
|
|
131
|
+
(domains ?? []).map(async (d) => {
|
|
132
|
+
const host = d.backendHost || '127.0.0.1';
|
|
133
|
+
const port = d.port || 80;
|
|
134
|
+
|
|
135
|
+
// A backend given as a URL is proxied somewhere we do not own; `up: null`
|
|
136
|
+
// is the contract's "not applicable", and it must not be notified about.
|
|
137
|
+
if (typeof host === 'string' && host.startsWith('http')) return { name: d.name, up: null };
|
|
138
|
+
|
|
139
|
+
const { up } = await probeBackend(host, port);
|
|
140
|
+
return { name: d.name, up, port, host };
|
|
141
|
+
}),
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* The most recent sweep, kept so the Hub agent can report REAL health.
|
|
147
|
+
*
|
|
148
|
+
* The agent used to send `up: d.enabled` — the enabled flag dressed up as a
|
|
149
|
+
* health result — so the Hub showed every enabled domain as healthy whether
|
|
150
|
+
* its backend answered or not. It runs in the same process as these monitors
|
|
151
|
+
* (core/jobs/index.js starts both), so it reads the sweep rather than probing
|
|
152
|
+
* every backend a second time.
|
|
153
|
+
*/
|
|
154
|
+
let lastDomainHealth = null;
|
|
155
|
+
|
|
156
|
+
/** @returns {{ at: number, results: Array<Object> }|null} */
|
|
157
|
+
export function getLastDomainHealth() {
|
|
158
|
+
return lastDomainHealth;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Probes every configured domain, publishes the list, and notifies on each
|
|
163
|
+
* domain's transitions.
|
|
164
|
+
*
|
|
165
|
+
* @returns {Promise<Array<{ name: string, up: boolean|null, port?: number, host?: string }>>}
|
|
166
|
+
*/
|
|
167
|
+
export async function checkDomainHealth() {
|
|
168
|
+
const domains = dbGet('domains') || [];
|
|
169
|
+
|
|
170
|
+
const results = await probeDomains(domains);
|
|
171
|
+
lastDomainHealth = { at: Date.now(), results };
|
|
172
|
+
|
|
173
|
+
publish('domain:health', results);
|
|
174
|
+
|
|
175
|
+
for (const result of results) {
|
|
176
|
+
if (result.up === null) continue;
|
|
177
|
+
|
|
178
|
+
await dispatchTransition({
|
|
179
|
+
domain: result.name,
|
|
180
|
+
eventType: 'domain_health',
|
|
181
|
+
status: result.up ? 'up' : 'down',
|
|
182
|
+
buildEvent: (d) => (d.reason === 'recovery'
|
|
183
|
+
? createDomainRecoveredEvent(result.name, result.port)
|
|
184
|
+
: createDomainDownEvent(result.name, result.port)),
|
|
185
|
+
emitLocal: (event) => publish('notification:new', event),
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
return results;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// ─── certificate expiry ───────────────────────────────────────────────────────
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Warns about certificates close to expiry.
|
|
196
|
+
*
|
|
197
|
+
* Expiry is NOT a state transition, so it does not go through the claim path:
|
|
198
|
+
* it is time-based, and the same warning is meant to repeat on each check
|
|
199
|
+
* until the certificate is renewed. The dedupe that matters here is "once per
|
|
200
|
+
* check interval", which the 12-hour timer already provides.
|
|
201
|
+
*
|
|
202
|
+
* That does mean a daemon and a dashboard both running this would double a
|
|
203
|
+
* warning — which is exactly why only one process owns the jobs. The owner
|
|
204
|
+
* election in core/jobs/index.js is what prevents it, not a per-event claim.
|
|
205
|
+
*
|
|
206
|
+
* @returns {Promise<Array<Object>>} The events that were published
|
|
207
|
+
*/
|
|
208
|
+
export async function checkCertExpiry() {
|
|
209
|
+
const events = [];
|
|
210
|
+
|
|
211
|
+
let certs;
|
|
212
|
+
try {
|
|
213
|
+
certs = await listAllCerts();
|
|
214
|
+
} catch {
|
|
215
|
+
// A missing or unreadable cert directory is normal on a fresh install.
|
|
216
|
+
return events;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const config = loadConfig();
|
|
220
|
+
const settings = getAutoRenewSettings(config);
|
|
221
|
+
// Migrated records, the same ones the renewal job reads, so the two cannot
|
|
222
|
+
// reach different conclusions about whether a certificate is in use.
|
|
223
|
+
const domains = getDomains();
|
|
224
|
+
|
|
225
|
+
for (const cert of certs) {
|
|
226
|
+
if (cert.daysLeft === null || cert.daysLeft >= CERT_WARN_DAYS) continue;
|
|
227
|
+
|
|
228
|
+
// What auto-renewal intends for this certificate decides what the warning
|
|
229
|
+
// should say, and whether it is worth sending yet.
|
|
230
|
+
let plan = 'off';
|
|
231
|
+
try {
|
|
232
|
+
const info = await describeCertificate(cert.domain);
|
|
233
|
+
plan = autoRenewPlan({ certDomain: cert.domain, info, settings, domains, sslDir: config.sslDir });
|
|
234
|
+
} catch { /* unreadable: warn as before */ }
|
|
235
|
+
|
|
236
|
+
// A certificate that WILL be renewed in the background, and has not
|
|
237
|
+
// reached its renewal window yet, is not a problem to report: warning
|
|
238
|
+
// twice a day for three weeks about something that is already handled
|
|
239
|
+
// only teaches people to ignore the channel. Once it is inside the window
|
|
240
|
+
// and still not renewed, the renewal has failed and the warning returns
|
|
241
|
+
// alongside the failure notification.
|
|
242
|
+
if (plan === 'auto' && cert.daysLeft > settings.days) continue;
|
|
243
|
+
|
|
244
|
+
// The socket layer built this event inline, duplicating a factory that
|
|
245
|
+
// already existed in core/events.js with the identical shape. Using the
|
|
246
|
+
// factory keeps the payload in one place, next to the others the contract
|
|
247
|
+
// documents.
|
|
248
|
+
const event = createCertExpiryEvent(cert.domain, cert.daysLeft, {
|
|
249
|
+
manualRenewal: plan === 'manual',
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
publish('notification:new', event);
|
|
253
|
+
events.push(event);
|
|
254
|
+
|
|
255
|
+
try {
|
|
256
|
+
await sendNotification(event, getEffectiveChannelIds(cert.domain, 'cert_expiry'));
|
|
257
|
+
} catch (err) {
|
|
258
|
+
console.error('[jobs/monitors] cert expiry notification failed:', err.message);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
return events;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// ─── certificate renewal ──────────────────────────────────────────────────────
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Renews HTTP-01 certificates that are inside the configured window
|
|
269
|
+
* (`sslAutoRenew`, `sslAutoRenewDays` — 7 days by default).
|
|
270
|
+
*
|
|
271
|
+
* The decisions live in core/cert-renewal.js; this only gives the job a place
|
|
272
|
+
* in the schedule and routes its notifications the same way the expiry warning
|
|
273
|
+
* is routed, so a domain's `cert_expiry` channels receive "renewed" and
|
|
274
|
+
* "renewal failed" without any new configuration.
|
|
275
|
+
*
|
|
276
|
+
* @returns {Promise<Object>} runAutoRenewal()'s summary
|
|
277
|
+
*/
|
|
278
|
+
export async function checkCertRenewal() {
|
|
279
|
+
return runAutoRenewal({
|
|
280
|
+
onEvent: async (event, domain) => {
|
|
281
|
+
publish('notification:new', event);
|
|
282
|
+
await sendNotification(event, getEffectiveChannelIds(domain, 'cert_expiry'));
|
|
283
|
+
},
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// ─── lifecycle ────────────────────────────────────────────────────────────────
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Starts every monitor and returns a handle that stops them.
|
|
291
|
+
*
|
|
292
|
+
* Each tick is guarded against overlapping with itself: a domain-health sweep
|
|
293
|
+
* over many domains can outlast its interval, and starting a second sweep
|
|
294
|
+
* while the first is still probing would double the work and let two
|
|
295
|
+
* transitions for the same domain be evaluated concurrently.
|
|
296
|
+
*
|
|
297
|
+
* @param {Object} [opts]
|
|
298
|
+
* @param {boolean} [opts.runImmediately] Poll once on start (default true)
|
|
299
|
+
* @param {Object} [opts.intervals] Override the intervals, for tests
|
|
300
|
+
* @returns {{ stop: () => void, tick: () => Promise<void> }}
|
|
301
|
+
*/
|
|
302
|
+
export function startMonitors({ runImmediately = true, intervals = {} } = {}) {
|
|
303
|
+
const every = {
|
|
304
|
+
nginx: intervals.nginx ?? NGINX_POLL_MS,
|
|
305
|
+
domains: intervals.domains ?? DOMAIN_HEALTH_POLL_MS,
|
|
306
|
+
certs: intervals.certs ?? CERT_CHECK_MS,
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
const timers = [];
|
|
310
|
+
let stopped = false;
|
|
311
|
+
|
|
312
|
+
/** Wraps a monitor so it never overlaps itself and never throws into a timer. */
|
|
313
|
+
const guarded = (name, fn) => {
|
|
314
|
+
let running = false;
|
|
315
|
+
return async () => {
|
|
316
|
+
if (stopped || running) return;
|
|
317
|
+
running = true;
|
|
318
|
+
try {
|
|
319
|
+
await fn();
|
|
320
|
+
} catch (err) {
|
|
321
|
+
console.error(`[jobs/monitors] ${name} failed:`, err.message);
|
|
322
|
+
} finally {
|
|
323
|
+
running = false;
|
|
324
|
+
}
|
|
325
|
+
};
|
|
326
|
+
};
|
|
327
|
+
|
|
328
|
+
const runNginx = guarded('nginx status', checkNginx);
|
|
329
|
+
const runDomains = guarded('domain health', checkDomainHealth);
|
|
330
|
+
// Renewal first, then the expiry check: a certificate that was just renewed
|
|
331
|
+
// must not also be warned about in the same pass. Each half is isolated so a
|
|
332
|
+
// failure in renewal can never silence the warning it would have prevented.
|
|
333
|
+
const runCerts = guarded('certificates', async () => {
|
|
334
|
+
try {
|
|
335
|
+
await checkCertRenewal();
|
|
336
|
+
} catch (err) {
|
|
337
|
+
console.error('[jobs/monitors] cert renewal failed:', err.message);
|
|
338
|
+
}
|
|
339
|
+
await checkCertExpiry();
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
timers.push(setInterval(runNginx, every.nginx));
|
|
343
|
+
timers.push(setInterval(runDomains, every.domains));
|
|
344
|
+
timers.push(setInterval(runCerts, every.certs));
|
|
345
|
+
|
|
346
|
+
// unref() is wrong here: the daemon's whole purpose is to keep running, and
|
|
347
|
+
// the one process that must not be kept alive by these timers (a CLI that
|
|
348
|
+
// only reads status) does not start them at all.
|
|
349
|
+
|
|
350
|
+
const tick = async () => {
|
|
351
|
+
await runNginx();
|
|
352
|
+
await runDomains();
|
|
353
|
+
await runCerts();
|
|
354
|
+
};
|
|
355
|
+
|
|
356
|
+
if (runImmediately) {
|
|
357
|
+
tick().catch((err) => console.error('[jobs/monitors] first poll failed:', err.message));
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
return {
|
|
361
|
+
stop() {
|
|
362
|
+
stopped = true;
|
|
363
|
+
for (const t of timers) clearInterval(t);
|
|
364
|
+
timers.length = 0;
|
|
365
|
+
},
|
|
366
|
+
tick,
|
|
367
|
+
};
|
|
368
|
+
}
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* core/jobs/owner.js
|
|
3
|
+
*
|
|
4
|
+
* A marker naming the process that is currently RUNNING the background jobs.
|
|
5
|
+
*
|
|
6
|
+
* ── Why this is separate from the daemon lock ──────────────────────────────
|
|
7
|
+
*
|
|
8
|
+
* `core/daemon/lock.js` answers "is a daemon running?". That is not the same
|
|
9
|
+
* question as "who is running the jobs", because an open dashboard runs them
|
|
10
|
+
* whenever no daemon does. The two were conflated, and the gap showed up as a
|
|
11
|
+
* real fault:
|
|
12
|
+
*
|
|
13
|
+
* 1. a daemon starts and takes the daemon lock
|
|
14
|
+
* 2. the dashboard is still running the jobs — it only notices the new
|
|
15
|
+
* daemon on its next ownership poll, up to 15 seconds later
|
|
16
|
+
* 3. for those seconds BOTH processes have a Hub agent connected with the
|
|
17
|
+
* same serverId, and the Hub keeps one socket per server: it evicts them
|
|
18
|
+
* in turn, so the user sees "io server disconnect" twice
|
|
19
|
+
*
|
|
20
|
+
* The daemon lock cannot fix that on its own: the dashboard deliberately never
|
|
21
|
+
* takes it (otherwise `ezz daemon` could not start while a browser tab is
|
|
22
|
+
* open). So job ownership needs its own marker, which BOTH roles claim, and a
|
|
23
|
+
* starting daemon waits for the dashboard to release before it starts anything
|
|
24
|
+
* that talks to the outside world.
|
|
25
|
+
*
|
|
26
|
+
* ── The protocol ───────────────────────────────────────────────────────────
|
|
27
|
+
*
|
|
28
|
+
* - whoever runs the jobs holds `jobs.owner` and refreshes its heartbeat
|
|
29
|
+
* - a daemon takes the daemon lock FIRST (so the dashboard sees it at once,
|
|
30
|
+
* via fs.watch), then waits for `jobs.owner` to free up, then claims it
|
|
31
|
+
* - a dashboard releases it the moment it learns a daemon exists
|
|
32
|
+
*
|
|
33
|
+
* Staleness is judged exactly as the daemon lock judges it — heartbeat old AND
|
|
34
|
+
* pid gone — for the same reason: a live pid does not prove our process still
|
|
35
|
+
* owns it, and a stale heartbeat alone can be a paused process.
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
import fs from 'fs';
|
|
39
|
+
import path from 'path';
|
|
40
|
+
|
|
41
|
+
import { getDataDir } from '../platform.js';
|
|
42
|
+
import { isPidAlive } from '../daemon/lock.js';
|
|
43
|
+
|
|
44
|
+
/** How often the owner refreshes its heartbeat. */
|
|
45
|
+
export const OWNER_HEARTBEAT_MS = 5_000;
|
|
46
|
+
|
|
47
|
+
/** How old a heartbeat may get before the marker is a candidate for recovery. */
|
|
48
|
+
export const OWNER_STALE_AFTER_MS = 20_000;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* How long a starting daemon waits for a dashboard to let go.
|
|
52
|
+
*
|
|
53
|
+
* Generous enough for a dashboard that is mid-poll or mid-request, short
|
|
54
|
+
* enough that a daemon never appears to hang. If it expires the daemon starts
|
|
55
|
+
* anyway: being briefly doubled is better than a server with no monitoring at
|
|
56
|
+
* all, and the notification claim makes the overlap harmless.
|
|
57
|
+
*/
|
|
58
|
+
export const HANDOVER_WAIT_MS = 8_000;
|
|
59
|
+
|
|
60
|
+
const OWNER_FILE = 'jobs.owner';
|
|
61
|
+
|
|
62
|
+
/** @returns {string} Absolute path to the ownership marker. */
|
|
63
|
+
export function ownerPath(dataDir = getDataDir()) {
|
|
64
|
+
return path.join(dataDir, OWNER_FILE);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Reads the marker without judging it.
|
|
69
|
+
* @returns {{ pid: number, role: string|null, at: number, heartbeatAt: number }|null}
|
|
70
|
+
*/
|
|
71
|
+
export function readOwner(dataDir = getDataDir()) {
|
|
72
|
+
try {
|
|
73
|
+
const parsed = JSON.parse(fs.readFileSync(ownerPath(dataDir), 'utf8'));
|
|
74
|
+
if (!Number.isInteger(parsed?.pid)) return null;
|
|
75
|
+
return {
|
|
76
|
+
pid: parsed.pid,
|
|
77
|
+
role: parsed.role ?? null,
|
|
78
|
+
at: Number(parsed.at) || 0,
|
|
79
|
+
heartbeatAt: Number(parsed.heartbeatAt) || 0,
|
|
80
|
+
};
|
|
81
|
+
} catch {
|
|
82
|
+
// Missing, unreadable or half-written all mean "nobody is holding it".
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Who holds the jobs, as far as anyone else can tell.
|
|
89
|
+
*
|
|
90
|
+
* @param {Object} [opts]
|
|
91
|
+
* @returns {{ held: boolean, mine: boolean, stale: boolean, owner: Object|null, reason: string }}
|
|
92
|
+
*/
|
|
93
|
+
export function inspect({ dataDir = getDataDir(), now = Date.now() } = {}) {
|
|
94
|
+
const owner = readOwner(dataDir);
|
|
95
|
+
if (!owner) return { held: false, mine: false, stale: false, owner: null, reason: 'nobody holds the jobs' };
|
|
96
|
+
|
|
97
|
+
const mine = owner.pid === process.pid;
|
|
98
|
+
const age = now - owner.heartbeatAt;
|
|
99
|
+
const alive = isPidAlive(owner.pid);
|
|
100
|
+
const fresh = age < OWNER_STALE_AFTER_MS;
|
|
101
|
+
|
|
102
|
+
if (alive || fresh) {
|
|
103
|
+
return {
|
|
104
|
+
held: true,
|
|
105
|
+
mine,
|
|
106
|
+
stale: false,
|
|
107
|
+
owner,
|
|
108
|
+
reason: mine
|
|
109
|
+
? 'this process holds the jobs'
|
|
110
|
+
: `pid ${owner.pid} (${owner.role ?? 'unknown role'}) holds the jobs`,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return {
|
|
115
|
+
held: false,
|
|
116
|
+
mine: false,
|
|
117
|
+
stale: true,
|
|
118
|
+
owner,
|
|
119
|
+
reason: `pid ${owner.pid} is gone and its heartbeat is ${Math.round(age / 1000)}s old`,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Takes ownership of the jobs.
|
|
125
|
+
*
|
|
126
|
+
* @param {Object} opts
|
|
127
|
+
* @param {string} [opts.dataDir]
|
|
128
|
+
* @param {string} opts.role 'daemon' | 'dashboard'
|
|
129
|
+
* @returns {{ ok: true, recovered: boolean } | { ok: false, reason: string, owner: Object|null }}
|
|
130
|
+
*/
|
|
131
|
+
export function claim({ dataDir = getDataDir(), role = 'unknown', now = Date.now() } = {}) {
|
|
132
|
+
const state = inspect({ dataDir, now });
|
|
133
|
+
|
|
134
|
+
if (state.held && !state.mine) {
|
|
135
|
+
return { ok: false, reason: state.reason, owner: state.owner };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
write({ dataDir, role, now, at: state.mine ? state.owner.at : now });
|
|
139
|
+
return { ok: true, recovered: state.stale };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Refreshes the heartbeat.
|
|
144
|
+
*
|
|
145
|
+
* @returns {boolean} True while this process still owns the jobs
|
|
146
|
+
*/
|
|
147
|
+
export function heartbeat({ dataDir = getDataDir(), role = 'unknown', now = Date.now() } = {}) {
|
|
148
|
+
const current = readOwner(dataDir);
|
|
149
|
+
if (!current || current.pid !== process.pid) return false;
|
|
150
|
+
|
|
151
|
+
write({ dataDir, role, now, at: current.at });
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Lets the jobs go, but only if this process holds them.
|
|
157
|
+
* @returns {boolean} True when the marker was actually removed
|
|
158
|
+
*/
|
|
159
|
+
export function release({ dataDir = getDataDir() } = {}) {
|
|
160
|
+
const current = readOwner(dataDir);
|
|
161
|
+
if (!current || current.pid !== process.pid) return false;
|
|
162
|
+
|
|
163
|
+
try {
|
|
164
|
+
fs.unlinkSync(ownerPath(dataDir));
|
|
165
|
+
return true;
|
|
166
|
+
} catch {
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Waits for whoever holds the jobs to let go.
|
|
173
|
+
*
|
|
174
|
+
* Used by a starting daemon, after it has taken the daemon lock. The lock is
|
|
175
|
+
* what tells the dashboard to stand down; this is what makes the daemon wait
|
|
176
|
+
* until it actually has.
|
|
177
|
+
*
|
|
178
|
+
* @param {Object} [opts]
|
|
179
|
+
* @returns {Promise<{ free: boolean, waitedMs: number, reason: string }>}
|
|
180
|
+
*/
|
|
181
|
+
export async function waitUntilFree({
|
|
182
|
+
dataDir = getDataDir(),
|
|
183
|
+
timeoutMs = HANDOVER_WAIT_MS,
|
|
184
|
+
everyMs = 100,
|
|
185
|
+
} = {}) {
|
|
186
|
+
const started = Date.now();
|
|
187
|
+
const deadline = started + timeoutMs;
|
|
188
|
+
|
|
189
|
+
for (;;) {
|
|
190
|
+
const state = inspect({ dataDir });
|
|
191
|
+
if (!state.held || state.mine) {
|
|
192
|
+
return { free: true, waitedMs: Date.now() - started, reason: state.reason };
|
|
193
|
+
}
|
|
194
|
+
if (Date.now() >= deadline) {
|
|
195
|
+
return { free: false, waitedMs: Date.now() - started, reason: state.reason };
|
|
196
|
+
}
|
|
197
|
+
await new Promise((r) => setTimeout(r, everyMs));
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Writes the marker atomically.
|
|
203
|
+
*
|
|
204
|
+
* Same temp-file-and-rename as the daemon lock: a reader must never see a
|
|
205
|
+
* half-written marker and conclude nobody owns the jobs.
|
|
206
|
+
*/
|
|
207
|
+
function write({ dataDir, role, now, at }) {
|
|
208
|
+
const payload = { pid: process.pid, role, at: at ?? now, heartbeatAt: now };
|
|
209
|
+
const target = ownerPath(dataDir);
|
|
210
|
+
const temp = `${target}.${process.pid}.tmp`;
|
|
211
|
+
|
|
212
|
+
fs.mkdirSync(dataDir, { recursive: true });
|
|
213
|
+
fs.writeFileSync(temp, `${JSON.stringify(payload)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
214
|
+
fs.renameSync(temp, target);
|
|
215
|
+
|
|
216
|
+
return payload;
|
|
217
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* core/jobs/revisionWatcher.js
|
|
3
|
+
*
|
|
4
|
+
* Watches the shared change counter in the database and publishes a
|
|
5
|
+
* `local:changed` event when another process moved it.
|
|
6
|
+
*
|
|
7
|
+
* Why this exists: the CLI, the dashboard and the daemon are separate
|
|
8
|
+
* processes. When `ezz` adds a domain, a dashboard tab that is already open
|
|
9
|
+
* has no way to find out — there is no shared memory and no message passing
|
|
10
|
+
* between them. `hub-revision` in the database is the cheapest signal that
|
|
11
|
+
* works for every combination: one process bumps it, the others notice.
|
|
12
|
+
*
|
|
13
|
+
* The hub agent already polls the same counter for its own purposes (pushing
|
|
14
|
+
* changes to the Hub). This watcher is the local half of that: it tells open
|
|
15
|
+
* dashboards to refresh. Both read the counter; neither writes it.
|
|
16
|
+
*
|
|
17
|
+
* ── Why a poll and not fs.watch ────────────────────────────────────────────
|
|
18
|
+
*
|
|
19
|
+
* The counter is a row inside the SQLite file, and a file watcher on a SQLite
|
|
20
|
+
* database reports write-ahead-log churn rather than logical changes — it
|
|
21
|
+
* fires when nothing we care about moved, and can miss a change that was
|
|
22
|
+
* already flushed. A 3-second read of one integer is far cheaper than getting
|
|
23
|
+
* that wrong. `fs.watch` is used elsewhere for the daemon lock, which really
|
|
24
|
+
* is one file whose existence is the signal.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { getHubRevision } from '../hubLink.js';
|
|
28
|
+
import { publish } from './bus.js';
|
|
29
|
+
|
|
30
|
+
/** How often the counter is read. */
|
|
31
|
+
export const REVISION_POLL_MS = 3_000;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Starts watching. The first read only establishes a baseline — it never
|
|
35
|
+
* publishes, or every start-up would tell the UI that everything changed.
|
|
36
|
+
*
|
|
37
|
+
* @param {Object} [opts]
|
|
38
|
+
* @param {number} [opts.intervalMs]
|
|
39
|
+
* @returns {{ stop: () => void, check: () => void, revision: () => number }}
|
|
40
|
+
*/
|
|
41
|
+
export function startRevisionWatcher({ intervalMs = REVISION_POLL_MS, onChange = null } = {}) {
|
|
42
|
+
let last = safeRevision() ?? 0;
|
|
43
|
+
|
|
44
|
+
const check = () => {
|
|
45
|
+
const current = safeRevision();
|
|
46
|
+
// A failed read is not a change. Returning 0 here instead of null would
|
|
47
|
+
// read as the counter going backwards and publish a spurious refresh every
|
|
48
|
+
// time the database was momentarily locked by another process's write.
|
|
49
|
+
if (current === null || current === last) return;
|
|
50
|
+
|
|
51
|
+
// Going DOWN means the database was replaced or reset — a restore, or a
|
|
52
|
+
// fresh install over the top. Treat it as a change too: the UI's cached
|
|
53
|
+
// view is wrong either way, and refusing to refresh would leave it stale
|
|
54
|
+
// for ever.
|
|
55
|
+
const direction = current > last ? 'changed' : 'reset';
|
|
56
|
+
last = current;
|
|
57
|
+
|
|
58
|
+
// Section 'domains' is the broadest useful hint: the counter is bumped for
|
|
59
|
+
// domains, certificates and permissions, and the UI refreshes the section
|
|
60
|
+
// it is shown. A more precise signal would need the writer to record what
|
|
61
|
+
// it changed, which is a change to every writer for little gain.
|
|
62
|
+
publish('local:changed', {
|
|
63
|
+
section: 'domains',
|
|
64
|
+
source: 'revision',
|
|
65
|
+
at: Date.now(),
|
|
66
|
+
revision: current,
|
|
67
|
+
direction,
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
// The job owner also uses this to reconcile things that live outside the
|
|
71
|
+
// UI — the hub agent in particular, since pairing changes are made in the
|
|
72
|
+
// dashboard or the CLI but the agent belongs to whichever process owns the
|
|
73
|
+
// jobs. A throw here must not stop the watcher: the next change retries.
|
|
74
|
+
if (onChange) {
|
|
75
|
+
try {
|
|
76
|
+
onChange({ revision: current, direction });
|
|
77
|
+
} catch (err) {
|
|
78
|
+
console.error('[revisionWatcher] change handler threw:', err.message);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
const timer = setInterval(check, intervalMs);
|
|
84
|
+
|
|
85
|
+
return {
|
|
86
|
+
stop() { clearInterval(timer); },
|
|
87
|
+
check,
|
|
88
|
+
revision: () => last,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Reading the counter must never kill the watcher: a locked database during
|
|
94
|
+
* another process's write is transient, and the next tick will succeed.
|
|
95
|
+
*
|
|
96
|
+
* @returns {number|null} null when the read failed — NOT 0, which is a real
|
|
97
|
+
* revision value and would be mistaken for the counter being reset.
|
|
98
|
+
*/
|
|
99
|
+
function safeRevision() {
|
|
100
|
+
try {
|
|
101
|
+
const n = getHubRevision();
|
|
102
|
+
return typeof n === 'number' ? n : null;
|
|
103
|
+
} catch {
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
}
|