@twinsearth/w2m-dsh-plugin 0.5.9 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +288 -3
- package/PROTOCOL-v0.4.0.md +11 -3
- package/PROTOCOL.md +69 -15
- package/README.md +180 -27
- package/SECURITY.md +69 -19
- package/TROUBLESHOOTING.md +100 -53
- package/bin/w2m-rabbit.mjs +32 -5
- package/deploy/networking/LESSONS.md +18 -1
- package/deploy/networking/README.md +32 -7
- package/deploy/windows/README.md +20 -1
- package/deploy/windows/install-service.ps1 +98 -19
- package/deploy/windows/verify-supervisor.mjs +794 -0
- package/deploy/windows/w2m-agent-supervisor.mjs +867 -40
- package/docs/AUDIT.md +10 -1
- package/docs/DEPLOY.md +16 -3
- package/docs/P2P-CRYPTO.md +105 -4
- package/docs/PIPELINE-DESIGN.md +35 -10
- package/docs/TROUBLESHOOTING.md +22 -3
- package/package.json +5 -5
- package/src/agent/agent.mjs +152 -5
- package/src/agent/allowed-commands.mjs +480 -18
- package/src/agent/crypto.mjs +238 -40
- package/src/agent/p2p-node.mjs +40 -0
- package/src/agent/p2p.mjs +323 -30
- package/src/agent/resolve.mjs +68 -3
- package/src/plugin/fleet-target.mjs +160 -15
- package/src/plugin/tools.mjs +27 -29
- package/src/relay/server.mjs +74 -11
|
@@ -33,9 +33,34 @@
|
|
|
33
33
|
// (cmd, PowerShell 5.1, Task Scheduler). A config file has no quoting rules at all, it is one more
|
|
34
34
|
// thing the installer can write atomically, and it stays readable for whoever debugs the service.
|
|
35
35
|
//
|
|
36
|
+
// ONE SUPERVISOR, ONE AGENT -- WHY THE LOCK EXISTS
|
|
37
|
+
//
|
|
38
|
+
// Measured on this machine, over one afternoon: the watchdog trigger restarted the supervisor every
|
|
39
|
+
// five minutes while the previous generation's *detached* agent was still alive, and the start-time
|
|
40
|
+
// reap below could only ever see the single pid the heartbeat remembered. Fourteen `w2m` processes
|
|
41
|
+
// accumulated, nine of them `w2m-localside` agents sharing one identity. The punch port (41235) was
|
|
42
|
+
// held by an agent started thirteen hours earlier, so every later agent logged
|
|
43
|
+
// `P2P_BIND_FAILED: bind EADDRINUSE 0.0.0.0:41235`, the relay saw `streams: 9`, and every dispatch
|
|
44
|
+
// fell back to the relay with `P2P_UNREACHABLE`. The machine was on the fleet and useless for
|
|
45
|
+
// fourteen hours, and nothing in the heartbeat said so.
|
|
46
|
+
//
|
|
47
|
+
// The lock at the top of this file is what makes "one supervisor, one agent" true *by construction*
|
|
48
|
+
// rather than by remembering to clean up: a second supervisor generation cannot start work at all.
|
|
36
49
|
// Nothing here is Windows-specific except the defaults; the same supervisor works on POSIX hosts.
|
|
37
|
-
import { spawn } from 'node:child_process';
|
|
38
|
-
import {
|
|
50
|
+
import { spawn, spawnSync } from 'node:child_process';
|
|
51
|
+
import { createSocket } from 'node:dgram';
|
|
52
|
+
import {
|
|
53
|
+
appendFileSync,
|
|
54
|
+
closeSync,
|
|
55
|
+
existsSync,
|
|
56
|
+
mkdirSync,
|
|
57
|
+
openSync,
|
|
58
|
+
readFileSync,
|
|
59
|
+
renameSync,
|
|
60
|
+
rmSync,
|
|
61
|
+
writeFileSync,
|
|
62
|
+
writeSync,
|
|
63
|
+
} from 'node:fs';
|
|
39
64
|
import { dirname, join } from 'node:path';
|
|
40
65
|
|
|
41
66
|
/** Minimal `--flag value` / `--flag` parser: no dependency, no surprises. */
|
|
@@ -147,17 +172,253 @@ if (!existsSync(bin)) {
|
|
|
147
172
|
process.exit(78); // EX_CONFIG: a service manager should not keep retrying this
|
|
148
173
|
}
|
|
149
174
|
|
|
150
|
-
|
|
151
|
-
|
|
175
|
+
// Where this generation's lock, heartbeat and sidecar state live. The log's directory is the run
|
|
176
|
+
// directory: everything the installer wrote for one service sits together, and two services with
|
|
177
|
+
// different logs are two unrelated installations that must not share a lock.
|
|
178
|
+
const PROC_MARKER_AGENT = 'w2m-localside';
|
|
179
|
+
const PROC_MARKER_SUPERVISOR = 'w2m-agent-supervisor';
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Does this command line belong to a supervisor of THIS installation?
|
|
183
|
+
*
|
|
184
|
+
* Used for two decisions that must agree with each other: whether a pid recorded in the lock is still
|
|
185
|
+
* a supervisor (the lock may be taken over otherwise), and whether a process of this identity is an
|
|
186
|
+
* agent to reap or a supervisor to leave alone. A supervisor's command line carries the agent stub's
|
|
187
|
+
* path -- `--bin ...\w2m-localside-stub.mjs` -- so "mentions w2m-localside" is true of a supervisor,
|
|
188
|
+
* and a reap that matched on identity alone would kill its own program. Measured: it did.
|
|
189
|
+
*/
|
|
190
|
+
function isSupervisorCommandLine(cmd) {
|
|
191
|
+
return (
|
|
192
|
+
typeof cmd === 'string' &&
|
|
193
|
+
cmd.includes(PROC_MARKER_SUPERVISOR) &&
|
|
194
|
+
cmd.includes('--config') &&
|
|
195
|
+
(configPath === null || cmd.includes(configPath))
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
const runDir = dirname(logPath);
|
|
199
|
+
mkdirSync(runDir, { recursive: true });
|
|
152
200
|
// A second handle on the same file, as a descriptor: this is what the detached agent writes into,
|
|
153
201
|
// because a descriptor is not connected to this process's console or to any pipe of ours.
|
|
154
202
|
const logFd = openSync(logPath, 'a');
|
|
155
203
|
const stamp = () => new Date().toISOString();
|
|
204
|
+
/**
|
|
205
|
+
* One line, two destinations, both synchronous, and both for a measured reason.
|
|
206
|
+
*
|
|
207
|
+
* The log is written with `appendFileSync` rather than through a `WriteStream`: a stream buffers, and
|
|
208
|
+
* the generation that most needs to explain itself is the one that exits immediately -- the watchdog
|
|
209
|
+
* tick that finds the lock held prints its no-op line and exits, while a buffered write is still in
|
|
210
|
+
* memory and is discarded with the process. Measured: the line was missing from the log, so the log
|
|
211
|
+
* said nothing about the most common thing this file now does.
|
|
212
|
+
*/
|
|
156
213
|
const say = (line) => {
|
|
157
214
|
console.log(line);
|
|
158
|
-
|
|
215
|
+
try {
|
|
216
|
+
appendFileSync(logPath, `${line}\n`);
|
|
217
|
+
} catch {
|
|
218
|
+
/* the console line is still out; a log that cannot be written must not stop the supervisor */
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
/**
|
|
222
|
+
* An opt-in trace of the decisions that are otherwise only visible as their consequences, written
|
|
223
|
+
* as a JSON line per event. `verify-supervisor.mjs` sets `W2M_SUPERVISOR_TRACE` when it needs to
|
|
224
|
+
* explain a failure; nothing in a normal service run sets it, and an unset variable costs one
|
|
225
|
+
* comparison.
|
|
226
|
+
*/
|
|
227
|
+
const tracePath = typeof process.env.W2M_SUPERVISOR_TRACE === 'string' && process.env.W2M_SUPERVISOR_TRACE !== ''
|
|
228
|
+
? process.env.W2M_SUPERVISOR_TRACE
|
|
229
|
+
: null;
|
|
230
|
+
const traceFd = tracePath === null ? null : openSync(tracePath, 'a');
|
|
231
|
+
function trace(event, detail) {
|
|
232
|
+
if (traceFd === null) return;
|
|
233
|
+
try {
|
|
234
|
+
writeSync(traceFd, `${JSON.stringify({ at: stamp(), event, ...detail })}\n`);
|
|
235
|
+
} catch {
|
|
236
|
+
/* a trace that cannot be written must never be a reason to stop supervising */
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
// The last line of defence: a process that vanishes without a word is the exact failure this file
|
|
240
|
+
// exists to make impossible, so nothing is allowed to end it silently.
|
|
241
|
+
process.on('uncaughtException', (error) => {
|
|
242
|
+
trace('uncaught-exception', { message: error?.message ?? String(error), stack: error?.stack ?? null });
|
|
243
|
+
say(`[w2m-supervisor] uncaught exception: ${error?.stack ?? error?.message ?? String(error)}`);
|
|
244
|
+
});
|
|
245
|
+
process.on('unhandledRejection', (reason) => {
|
|
246
|
+
trace('unhandled-rejection', { message: reason?.message ?? String(reason), stack: reason?.stack ?? null });
|
|
247
|
+
say(`[w2m-supervisor] unhandled rejection: ${reason?.stack ?? reason?.message ?? String(reason)}`);
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
// ---------------------------------------------------------------------------------------------
|
|
251
|
+
// THE SINGLE-INSTANCE LOCK
|
|
252
|
+
//
|
|
253
|
+
// The watchdog trigger in `install-service.ps1` exists because a console-teardown death cannot be
|
|
254
|
+
// logged from inside the process it kills: the task must be able to say "be running, or start".
|
|
255
|
+
// That trigger is only safe if starting a *second* generation while a healthy one runs is a no-op,
|
|
256
|
+
// and that is exactly what the lock buys. Without it the trigger is a process factory: every
|
|
257
|
+
// restart of a supervisor whose agent survived leaves another agent behind, and the old reap --
|
|
258
|
+
// which trusted the single `child_pid` in the heartbeat -- could never catch up.
|
|
259
|
+
//
|
|
260
|
+
// `wx` is the atomic part: `open(..., 'wx')` is O_CREAT|O_EXCL, so of two generations racing to
|
|
261
|
+
// start, exactly one creates the file and the other gets EEXIST. The file *is* the lock only until
|
|
262
|
+
// its holder dies, though, and this project has measured two ways for that to happen abruptly:
|
|
263
|
+
// `Stop-Process -Force`, and a console teardown that kills the tree with 0xC000013A. Neither runs
|
|
264
|
+
// any handler here, so the file is simply left behind. It is therefore not the existence of the
|
|
265
|
+
// file that decides anything -- it is the liveness of the pid written in it. That is why a stale
|
|
266
|
+
// lock (holder gone, or the file unreadable) is taken over loudly instead of blocking the machine
|
|
267
|
+
// forever, and why `orphans_reaped`/`lock_takeovers` are in the heartbeat: taking a lock over is
|
|
268
|
+
// the normal aftermath of exactly the death this file exists to survive.
|
|
269
|
+
// ---------------------------------------------------------------------------------------------
|
|
270
|
+
const lockPath = typeof pick('lock', null) === 'string' ? pick('lock', null) : join(runDir, 'supervisor.lock');
|
|
271
|
+
/** Takeovers, cumulative across generations: kept beside the lock so a restart does not erase it. */
|
|
272
|
+
const lockStatePath = `${lockPath}.state.json`;
|
|
273
|
+
|
|
274
|
+
function readLockState(file_) {
|
|
275
|
+
try {
|
|
276
|
+
const parsed = JSON.parse(readFileSync(file_, 'utf8').replace(/^\uFEFF/, ''));
|
|
277
|
+
return parsed !== null && typeof parsed === 'object' ? parsed : {};
|
|
278
|
+
} catch {
|
|
279
|
+
return {};
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** Best effort, and only used for numbers: an unreadable sidecar is not a reason to refuse to run. */
|
|
284
|
+
function bumpLockTakeovers() {
|
|
285
|
+
try {
|
|
286
|
+
const previous = Number(readLockState(lockStatePath).takeovers ?? 0);
|
|
287
|
+
const next = Number.isFinite(previous) ? previous + 1 : 1;
|
|
288
|
+
const tmp = `${lockStatePath}.tmp`;
|
|
289
|
+
writeFileSync(tmp, `${JSON.stringify({ takeovers: next, updated_at: stamp() }, null, 2)}\n`, 'utf8');
|
|
290
|
+
renameSync(tmp, lockStatePath);
|
|
291
|
+
return next;
|
|
292
|
+
} catch {
|
|
293
|
+
return null;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const takeoversTotal = Number(lockStatePath === '' ? 0 : readLockState(lockStatePath).takeovers ?? 0) || 0;
|
|
298
|
+
const lockInfo = {
|
|
299
|
+
path: lockPath,
|
|
300
|
+
pid: process.pid,
|
|
301
|
+
started_at: stamp(),
|
|
302
|
+
takeovers_total: takeoversTotal,
|
|
303
|
+
takeovers_this_run: 0,
|
|
159
304
|
};
|
|
160
305
|
|
|
306
|
+
/**
|
|
307
|
+
* Is another supervisor generation working right now?
|
|
308
|
+
*
|
|
309
|
+
* Alive is not enough on its own: Windows reuses pids, and a lock left by a killed supervisor can
|
|
310
|
+
* name a pid that now belongs to something else entirely -- long-lived, unrelated, and none of this
|
|
311
|
+
* program's business. Treating that as "held" would wedge the machine until the process happened to
|
|
312
|
+
* exit, which is the failure this whole file is against. So the pid is checked for being a
|
|
313
|
+
* *supervisor*: node, running this script, with this `--config`. When the process table cannot be
|
|
314
|
+
* read the answer is "assume held", because the one irreversible mistake is a second agent -- the
|
|
315
|
+
* watchdog ticks again in five minutes, and the log line says which of the two happened.
|
|
316
|
+
*
|
|
317
|
+
* @returns {{held: boolean, reason: string}}
|
|
318
|
+
*/
|
|
319
|
+
function lockHolder() {
|
|
320
|
+
if (!existsSync(lockPath)) return { held: false, reason: 'lock file is gone (the holder removed it)' };
|
|
321
|
+
let pid = null;
|
|
322
|
+
let holderStartedAt = null;
|
|
323
|
+
let parseNote = 'unreadable';
|
|
324
|
+
try {
|
|
325
|
+
const parsed = JSON.parse(readFileSync(lockPath, 'utf8').replace(/^\uFEFF/, ''));
|
|
326
|
+
// A pid written as a string is still a pid: the field has been written by hand at least once.
|
|
327
|
+
if (Number.isInteger(Number(parsed?.pid)) && Number(parsed.pid) > 0) {
|
|
328
|
+
pid = Number(parsed.pid);
|
|
329
|
+
holderStartedAt = typeof parsed?.started_at === 'string' ? parsed.started_at : null;
|
|
330
|
+
parseNote = 'readable';
|
|
331
|
+
} else {
|
|
332
|
+
parseNote = 'readable but has no usable pid';
|
|
333
|
+
}
|
|
334
|
+
} catch (error) {
|
|
335
|
+
// Unreadable is treated exactly like a dead holder -- see the block comment above: a truncated
|
|
336
|
+
// or half-written lock must not be able to keep the fleet down with a stale claim on the port.
|
|
337
|
+
parseNote = `unreadable (${error.code ?? error.message})`;
|
|
338
|
+
}
|
|
339
|
+
if (pid === null) return { held: false, reason: parseNote };
|
|
340
|
+
if (pid === process.pid) return { held: false, reason: `this process (pid=${pid}) owns it` };
|
|
341
|
+
try {
|
|
342
|
+
process.kill(pid, 0);
|
|
343
|
+
} catch (error) {
|
|
344
|
+
return { held: false, reason: `pid=${pid} is not running (${error.code ?? error.message})` };
|
|
345
|
+
}
|
|
346
|
+
const rows = listProcesses();
|
|
347
|
+
if (rows !== null) {
|
|
348
|
+
const row = rows.find((candidate) => candidate.pid === pid);
|
|
349
|
+
if (row !== undefined && !isSupervisorCommandLine(row.cmd)) {
|
|
350
|
+
return {
|
|
351
|
+
held: false,
|
|
352
|
+
reason: `pid=${pid} is alive but is not a supervisor of this installation (${row.cmd.slice(0, 140)})`,
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
return { held: true, reason: `pid=${pid} is a supervisor running this config (lock written ${holderStartedAt ?? 'at an unknown time'}, ${parseNote})` };
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// The lock is the first thing this process does that can have a side effect, and the last thing it
|
|
360
|
+
// gives up. Everything above this line is argument/config parsing, which changes nothing.
|
|
361
|
+
let lockFd = null;
|
|
362
|
+
for (let attempt = 0; attempt < 3 && lockFd === null; attempt += 1) {
|
|
363
|
+
try {
|
|
364
|
+
lockFd = openSync(lockPath, 'wx');
|
|
365
|
+
} catch (error) {
|
|
366
|
+
if (error.code !== 'EEXIST') {
|
|
367
|
+
// EX_CONFIG, not a crash: no lock means two agents is possible, and running anyway would
|
|
368
|
+
// trade a visible failure for the exact invisible one this file exists to prevent.
|
|
369
|
+
console.error(`w2m-supervisor: cannot create the lock ${lockPath}: ${error.message}`);
|
|
370
|
+
process.exit(78);
|
|
371
|
+
}
|
|
372
|
+
const holder = lockHolder();
|
|
373
|
+
if (holder.held) {
|
|
374
|
+
say(`[w2m-supervisor ${stamp()}] another supervisor generation already holds the lock: ${holder.reason}`);
|
|
375
|
+
say(`[w2m-supervisor ${stamp()}] lock=${lockPath} -- exiting without spawning; the watchdog tick is a no-op`);
|
|
376
|
+
// Deliberately exit 0: a no-op start is the *designed* outcome of a watchdog tick, not a
|
|
377
|
+
// failure, and a non-zero code would make Task Scheduler's RestartOnFailure spin on it.
|
|
378
|
+
process.exit(0);
|
|
379
|
+
}
|
|
380
|
+
try {
|
|
381
|
+
rmSync(lockPath, { force: true });
|
|
382
|
+
say(
|
|
383
|
+
`[w2m-supervisor ${stamp()}] stale lock taken over: not held because ${holder.reason} -- ` +
|
|
384
|
+
`${lockPath} removed, this generation now owns it`,
|
|
385
|
+
);
|
|
386
|
+
const total = bumpLockTakeovers();
|
|
387
|
+
if (Number.isFinite(total)) {
|
|
388
|
+
lockInfo.takeovers_total = total;
|
|
389
|
+
lockInfo.takeovers_this_run += 1;
|
|
390
|
+
}
|
|
391
|
+
} catch (removeError) {
|
|
392
|
+
say(`[w2m-supervisor ${stamp()}] could not clear the stale lock ${lockPath}: ${removeError.message}`);
|
|
393
|
+
process.exit(78);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
if (lockFd === null) {
|
|
398
|
+
say(`[w2m-supervisor ${stamp()}] could not take the lock ${lockPath} after 3 attempts -- exiting without spawning`);
|
|
399
|
+
process.exit(0);
|
|
400
|
+
}
|
|
401
|
+
writeSync(lockFd, `${JSON.stringify({ pid: process.pid, started_at: lockInfo.started_at, argv: process.argv.join(' ') }, null, 2)}\n`);
|
|
402
|
+
// Closed immediately: the lock is the file's existence plus the liveness of the pid inside it, not
|
|
403
|
+
// an open descriptor -- `wx` is what makes the create atomic and that has already happened.
|
|
404
|
+
closeSync(lockFd);
|
|
405
|
+
|
|
406
|
+
let lockHeld = true;
|
|
407
|
+
/** Give the lock up on the way out, so the next generation does not have to call it stale. */
|
|
408
|
+
function dropLock(why) {
|
|
409
|
+
if (!lockHeld) return;
|
|
410
|
+
lockHeld = false;
|
|
411
|
+
try {
|
|
412
|
+
rmSync(lockPath, { force: true });
|
|
413
|
+
say(`[w2m-supervisor ${stamp()}] lock released (${why})`);
|
|
414
|
+
} catch (error) {
|
|
415
|
+
say(`[w2m-supervisor ${stamp()}] could not remove the lock on exit: ${error.message}`);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
say(`[w2m-supervisor ${stamp()}] supervising ${process.execPath} ${[bin, '--rabbit', rabbit, '--project', project, '--name', name, '--state', state, '--allowed-commands', allowed, ...passthrough].join(' ')}`);
|
|
420
|
+
say(`[w2m-supervisor] lock=${lockPath} holder=pid:${process.pid} takeovers_total=${lockInfo.takeovers_total}`);
|
|
421
|
+
|
|
161
422
|
const argv = [
|
|
162
423
|
bin,
|
|
163
424
|
'--rabbit', rabbit,
|
|
@@ -168,7 +429,6 @@ const argv = [
|
|
|
168
429
|
...passthrough,
|
|
169
430
|
];
|
|
170
431
|
|
|
171
|
-
say(`[w2m-supervisor ${stamp()}] supervising ${process.execPath} ${argv.join(' ')}`);
|
|
172
432
|
let stopping = false;
|
|
173
433
|
for (const signal of ['SIGINT', 'SIGTERM', 'SIGBREAK']) {
|
|
174
434
|
process.on(signal, () => {
|
|
@@ -179,6 +439,33 @@ for (const signal of ['SIGINT', 'SIGTERM', 'SIGBREAK']) {
|
|
|
179
439
|
// Declared before the heartbeat block, which reads it: a `let` referenced earlier in the same scope
|
|
180
440
|
// would be a temporal-dead-zone throw, not an undefined value.
|
|
181
441
|
let runs = 0;
|
|
442
|
+
/** The worker that is alive right now, tracked here so every heartbeat can name it. */
|
|
443
|
+
let currentChildPid = null;
|
|
444
|
+
let currentChildStartedAt = null;
|
|
445
|
+
/** Item 2 and item 3 facts, kept for the life of this generation and repeated in every heartbeat. */
|
|
446
|
+
let orphansReaped = 0;
|
|
447
|
+
let orphansReapedPids = [];
|
|
448
|
+
let lastReason = null;
|
|
449
|
+
|
|
450
|
+
// Extra agent flags this supervisor understands for its own bookkeeping. They are still passed to
|
|
451
|
+
// the agent (which ignores unknown `--flags`) and exist so `verify-supervisor.mjs` can drive this
|
|
452
|
+
// file against a stub worker without Windows Task Scheduler and without touching the live agent.
|
|
453
|
+
const reapMatch = typeof pick('reap-match', null) === 'string' ? pick('reap-match', null) : state;
|
|
454
|
+
/**
|
|
455
|
+
* The punch port, taken from the same place the agent takes it. `--p2p-port` arrives through
|
|
456
|
+
* `passthrough` (it is an agent flag, written as `agentArgs: "--p2p-mode auto --p2p-port 41235"` in
|
|
457
|
+
* the installer's config), with the flag/config forms as fallbacks for a direct command line.
|
|
458
|
+
* `0` means "let the OS choose" and is refused by the agent itself, so it is no port to gate on.
|
|
459
|
+
*/
|
|
460
|
+
const p2pPort = (() => {
|
|
461
|
+
const at = passthrough.indexOf('--p2p-port');
|
|
462
|
+
const value = at >= 0 ? passthrough[at + 1] : pick('p2p-port', null);
|
|
463
|
+
const parsed = Number(value);
|
|
464
|
+
return Number.isInteger(parsed) && parsed > 0 && parsed <= 65_535 ? parsed : null;
|
|
465
|
+
})();
|
|
466
|
+
const p2pMode = String(pick('p2p-mode', 'auto')).toLowerCase();
|
|
467
|
+
/** Item 3's facts, repeated in every heartbeat for the life of this generation. */
|
|
468
|
+
const portInfo = { configured: p2pPort, state: 'unknown', owner_pid: null, note: null };
|
|
182
469
|
|
|
183
470
|
// ---------------------------------------------------------------------------------------------
|
|
184
471
|
// HEARTBEAT AND DEGRADED MARKER
|
|
@@ -194,10 +481,15 @@ let runs = 0;
|
|
|
194
481
|
// backoff between runs, within the healthy band
|
|
195
482
|
// degraded the crash-loop breaker has tripped: too many exits inside the window, so the fleet
|
|
196
483
|
// should see a machine that is present but unhealthy rather than one that is gone
|
|
484
|
+
// stopped this generation has given up the lock and exited
|
|
485
|
+
//
|
|
486
|
+
// Two fields are worth reading before the state:
|
|
487
|
+
// orphans_reaped / orphans_reaped_pids what the start-time reap terminated, by pid
|
|
488
|
+
// port.state free | bound | bound-by-other | held | unknown
|
|
197
489
|
//
|
|
198
490
|
// Written atomically (temp file + rename) so a reader never sees half a JSON document.
|
|
199
491
|
// ---------------------------------------------------------------------------------------------
|
|
200
|
-
const heartbeatPath = join(
|
|
492
|
+
const heartbeatPath = join(runDir, 'supervisor-heartbeat.json');
|
|
201
493
|
const degradedAfter = Number.isFinite(Number(pick('degraded-after', 3))) ? Number(pick('degraded-after', 3)) : 3;
|
|
202
494
|
const degradedWindowMs = 10 * 60_000;
|
|
203
495
|
const degradedBackoffMs = Number.isFinite(Number(pick('degraded-backoff-ms', 300_000)))
|
|
@@ -207,18 +499,45 @@ const exitTimes = [];
|
|
|
207
499
|
|
|
208
500
|
function writeHeartbeat(patch) {
|
|
209
501
|
const recent = exitTimes.filter((at) => Date.now() - at < degradedWindowMs).length;
|
|
210
|
-
|
|
502
|
+
// With a live child the state is `running`, whoever is asking. The 30 s beat passes no state of its
|
|
503
|
+
// own, and defaulting it to `backoff` published "backoff" alongside a `child_pid` that named a live
|
|
504
|
+
// agent -- measured on this machine at 17:30, with pid 9860 serving `stream ready`. The one file an
|
|
505
|
+
// outside observer reads must not be the one file that contradicts itself.
|
|
506
|
+
const state = patch.state ?? (currentChildPid !== null ? 'running' : recent >= degradedAfter ? 'degraded' : 'backoff');
|
|
211
507
|
const record = {
|
|
212
508
|
updated_at: stamp(),
|
|
213
509
|
state,
|
|
214
510
|
supervisor_pid: process.pid,
|
|
215
511
|
run: patch.run ?? runs,
|
|
216
|
-
|
|
217
|
-
|
|
512
|
+
// `currentChildPid`, never `null`, unless the patch says otherwise: this field used to be
|
|
513
|
+
// written as `patch.child_pid ?? null` on the 30 s beat, so a healthy generation with a live
|
|
514
|
+
// agent published `child_pid: null` -- measured on this machine at 17:30 while pid 9860 was
|
|
515
|
+
// serving `stream ready`. A field that is null exactly when the agent is fine is worse than no
|
|
516
|
+
// field: it is what made the old single-pid reap look like it had nothing to do.
|
|
517
|
+
child_pid: patch.child_pid !== undefined ? patch.child_pid : currentChildPid,
|
|
518
|
+
child_started_at: patch.child_started_at !== undefined ? patch.child_started_at : currentChildStartedAt,
|
|
218
519
|
restarts_total: Math.max(0, runs - 1),
|
|
219
520
|
restarts_within_10min: recent,
|
|
220
521
|
last_exit_code: patch.last_exit_code ?? null,
|
|
221
522
|
last_exit_at: patch.last_exit_at ?? null,
|
|
523
|
+
// Why the last attempt did not end in a running agent, when it did not: one of
|
|
524
|
+
// P2P_PORT_HELD_BY_OTHER, P2P_PORT_NOT_BOUND, or null.
|
|
525
|
+
last_reason: patch.last_reason !== undefined ? patch.last_reason : lastReason,
|
|
526
|
+
// Item 2: what this generation reaped, by pid. Counted across generations in the log.
|
|
527
|
+
orphans_reaped: orphansReaped,
|
|
528
|
+
orphans_reaped_pids: orphansReapedPids,
|
|
529
|
+
// Item 1: who holds the single-instance lock, and how often a dead generation had to be
|
|
530
|
+
// cleaned out of the way.
|
|
531
|
+
lock: { path: lockInfo.path, pid: lockInfo.pid, takeovers_total: lockInfo.takeovers_total, takeovers_this_run: lockInfo.takeovers_this_run },
|
|
532
|
+
// Item 3: the punch port, which is the difference between a machine that is *on* the fleet and
|
|
533
|
+
// one that is merely *listed* on it.
|
|
534
|
+
port: {
|
|
535
|
+
configured: portInfo.configured,
|
|
536
|
+
mode: p2pMode,
|
|
537
|
+
state: portInfo.state,
|
|
538
|
+
owner_pid: portInfo.owner_pid,
|
|
539
|
+
note: portInfo.note,
|
|
540
|
+
},
|
|
222
541
|
// The known killer, recorded in decimal because that is how Node reports it: 3221225786 is
|
|
223
542
|
// 0xC000013A, STATUS_CONTROL_C_EXIT. Signed, it is -1073741510.
|
|
224
543
|
note: 'exit code 3221225786 means the console went away; it is not a crash of the agent logic',
|
|
@@ -235,52 +554,512 @@ function writeHeartbeat(patch) {
|
|
|
235
554
|
return record;
|
|
236
555
|
}
|
|
237
556
|
|
|
238
|
-
writeHeartbeat({ state: 'backoff' });
|
|
239
557
|
// ---------------------------------------------------------------------------------------------
|
|
240
|
-
//
|
|
558
|
+
// PROCESS IDENTITY: HOW A PROCESS IS RECOGNISED, AND WHY NOT BY A REMEMBERED PID
|
|
559
|
+
//
|
|
560
|
+
// The old reap read `child_pid` out of the heartbeat and killed that one pid. Measured cost: with
|
|
561
|
+
// nine agents alive, eight were invisible to it, so it terminated one and left the rest holding the
|
|
562
|
+
// identity and the punch port. A remembered pid is a fact about the past; what is needed is a fact
|
|
563
|
+
// about the present, so this enumerates the process table and matches on **identity**:
|
|
564
|
+
//
|
|
565
|
+
// * the process is node (`node.exe` or a `node` basename of the actual executable), AND
|
|
566
|
+
// * its command line contains `w2m-localside`, AND
|
|
567
|
+
// * its command line contains this installation's identity string -- the agent's `--state`
|
|
568
|
+
// directory by default, `--reap-match` when a test harness or a second checkout needs its own.
|
|
569
|
+
//
|
|
570
|
+
// Everything a reap touches therefore belongs to *this* agent's identity; an unrelated node process
|
|
571
|
+
// -- a test runner, an editor's language server, DSH itself -- cannot match, because it has neither
|
|
572
|
+
// the agent script name nor this state directory on its command line. The command line, rather than
|
|
573
|
+
// a pid file or a parent link, is the only one of those that survives both deaths this project has
|
|
574
|
+
// measured: `Stop-Process -Force` and a console teardown. A detached child is re-parented (or
|
|
575
|
+
// orphaned) by both, so the parent link is worthless exactly when it is needed, while `--state` was
|
|
576
|
+
// on the command line from the first generation to the last and is what one identity *means*.
|
|
577
|
+
//
|
|
578
|
+
// The second marker (`w2m-agent-supervisor` + this `--config`) identifies superseded supervisor
|
|
579
|
+
// generations. They are reported, and are not killed: a live supervisor's lock is authoritative (the
|
|
580
|
+
// lock already makes a second generation exit), while killing a *live* supervisor by command-line
|
|
581
|
+
// match would race its own spawn. Orphaned agents, which cannot hold a lock, are the leak that
|
|
582
|
+
// actually took the port, and they are killed.
|
|
583
|
+
// ---------------------------------------------------------------------------------------------
|
|
584
|
+
|
|
585
|
+
/** The process table, or null when it cannot be read (then the caller says so rather than guessing). */
|
|
586
|
+
function listProcesses() {
|
|
587
|
+
if (process.platform !== 'win32') {
|
|
588
|
+
// POSIX: `ps` is the table. Kept here so the same reap works on the hosts this file claims to
|
|
589
|
+
// support, but the argument shape is the only thing shared -- the matching above is identical.
|
|
590
|
+
const result = spawnSync('ps', ['-eo', 'pid=,args='], { encoding: 'utf8', windowsHide: true, timeout: 10_000 });
|
|
591
|
+
if (result.status !== 0 || typeof result.stdout !== 'string') return null;
|
|
592
|
+
return result.stdout
|
|
593
|
+
.split(/\r?\n/)
|
|
594
|
+
.map((line) => line.trim())
|
|
595
|
+
.filter(Boolean)
|
|
596
|
+
.map((line) => {
|
|
597
|
+
const at = line.indexOf(' ');
|
|
598
|
+
return { pid: Number(line.slice(0, at)), cmd: line.slice(at + 1) };
|
|
599
|
+
})
|
|
600
|
+
.filter((row) => Number.isInteger(row.pid));
|
|
601
|
+
}
|
|
602
|
+
// ONE query for the whole table, filtered in JavaScript. It used to ask PowerShell for the match
|
|
603
|
+
// and paid for it twice: every filter was another `Get-CimInstance` (each with a CIM round trip)
|
|
604
|
+
// and, worse, the `if`/`elseif` block was joined with `;`, which PowerShell reads as an empty
|
|
605
|
+
// statement where a block is required -- a parse error that `-ErrorAction SilentlyContinue` hid
|
|
606
|
+
// while returning nothing. Nothing is exactly what "no orphans to reap" looks like, so the reap
|
|
607
|
+
// silently did nothing. The identity strings travel in the environment (no quoting rules apply)
|
|
608
|
+
// and every match is made here, where a wrong answer is a visible test failure rather than a
|
|
609
|
+
// missing row.
|
|
610
|
+
const script = [
|
|
611
|
+
"$ErrorActionPreference='SilentlyContinue'",
|
|
612
|
+
"Get-CimInstance Win32_Process -Filter \"Name='node.exe'\" | Select-Object ProcessId,CommandLine | ConvertTo-Csv -NoTypeInformation",
|
|
613
|
+
"Write-Output '#end'",
|
|
614
|
+
].join('\n');
|
|
615
|
+
const result = spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], {
|
|
616
|
+
encoding: 'utf8',
|
|
617
|
+
windowsHide: true,
|
|
618
|
+
timeout: 20_000,
|
|
619
|
+
});
|
|
620
|
+
if (result.error || typeof result.stdout !== 'string' || !result.stdout.includes('#end')) return null;
|
|
621
|
+
const rows = [];
|
|
622
|
+
for (const line of result.stdout.split(/\r?\n/)) {
|
|
623
|
+
const trimmed = line.trim();
|
|
624
|
+
if (!trimmed.startsWith('"')) continue;
|
|
625
|
+
// CSV from PowerShell: '"pid","command line"', with embedded quotes doubled. The command line
|
|
626
|
+
// itself contains commas, so the pid is taken from the front and the remainder is re-joined
|
|
627
|
+
// rather than split into fields.
|
|
628
|
+
const parts = trimmed.split('","');
|
|
629
|
+
const pid = Number(parts[0].replace(/^"|"$/g, ''));
|
|
630
|
+
if (!Number.isInteger(pid) || pid <= 0) continue;
|
|
631
|
+
rows.push({ pid, cmd: parts.join('","').replace(/^"|"$/g, '').replace(/""/g, '"') });
|
|
632
|
+
}
|
|
633
|
+
trace('process-table', { rows: rows.length });
|
|
634
|
+
return rows;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
/** Every live process that is this installation's agent, whoever started it. */
|
|
638
|
+
function agentProcesses() {
|
|
639
|
+
const rows = listProcesses();
|
|
640
|
+
if (rows === null) return null;
|
|
641
|
+
const wanted = String(reapMatch).toLowerCase().replace(/\//g, '\\');
|
|
642
|
+
const matched = [];
|
|
643
|
+
for (const row of rows) {
|
|
644
|
+
if (typeof row.cmd !== 'string') continue;
|
|
645
|
+
// A supervisor is never an agent, and this one is never a candidate for anything. Both checks
|
|
646
|
+
// are load-bearing: the identity string is an argument of the supervisor as well as of the agent
|
|
647
|
+
// (`--state <dir>` was in this file's own command line), so matching on identity alone made the
|
|
648
|
+
// supervisor reap *itself* -- measured, and it died between `reap-before-kill` and the kill
|
|
649
|
+
// returning, which is the worst possible version of this bug: a process that kills the agent of
|
|
650
|
+
// a different installation, or itself, is worse than one that leaves orphans behind.
|
|
651
|
+
if (row.pid === process.pid) continue;
|
|
652
|
+
const identity = row.cmd.toLowerCase().replace(/\//g, '\\');
|
|
653
|
+
if (!identity.includes(wanted)) continue;
|
|
654
|
+
if (isSupervisorCommandLine(row.cmd)) {
|
|
655
|
+
trace('reap-skipped-supervisor', { pid: row.pid });
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
658
|
+
if (!row.cmd.toLowerCase().includes(PROC_MARKER_AGENT)) {
|
|
659
|
+
trace('reap-skipped-no-agent-marker', { pid: row.pid, cmd: row.cmd.slice(0, 160) });
|
|
660
|
+
continue;
|
|
661
|
+
}
|
|
662
|
+
matched.push(row);
|
|
663
|
+
}
|
|
664
|
+
trace('agent-processes', { rows: rows.length, wanted, matched: matched.map((row) => row.pid) });
|
|
665
|
+
return matched;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/** Superseded supervisor generations, for the report only -- see the block comment above. */
|
|
669
|
+
function supervisorProcesses() {
|
|
670
|
+
const rows = listProcesses();
|
|
671
|
+
if (rows === null) return null;
|
|
672
|
+
const self = process.pid;
|
|
673
|
+
return rows.filter((row) => row.pid !== self && isSupervisorCommandLine(row.cmd));
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
function isAlive(pid) {
|
|
677
|
+
try {
|
|
678
|
+
process.kill(pid, 0);
|
|
679
|
+
return true;
|
|
680
|
+
} catch {
|
|
681
|
+
return false;
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* Terminate one pid, hard on Windows -- and then *check*, because this is the operation that decides
|
|
687
|
+
* whether an orphan is really gone.
|
|
688
|
+
*
|
|
689
|
+
* `process.kill(pid, 'SIGKILL')` cannot end a process this one has no console relationship with, and
|
|
690
|
+
* a Task Scheduler agent has none, so `taskkill /T /F` is the Windows way to end a tree; `/T` matters
|
|
691
|
+
* for a grandchild an agent may have spawned. But `/T` alone is not enough, and its failure is
|
|
692
|
+
* silent: measured, `taskkill /PID <pid of an exited parent> /T /F` exits 128 with "process not
|
|
693
|
+
* found" and kills nothing at all -- and a detached agent is exactly a pid whose parent has already
|
|
694
|
+
* exited. A reap that trusted taskkill's exit code would report success while the orphan kept its
|
|
695
|
+
* claim on the punch port, which is this defect all over again. So: retry, escalate to a direct kill
|
|
696
|
+
* of the pid itself, and return the truth of `isAlive` rather than taskkill's opinion.
|
|
697
|
+
*/
|
|
698
|
+
function terminate(pid, timeoutMs = 2_000) {
|
|
699
|
+
// Never this process. `taskkill /PID <self>` succeeds, so a matching bug upstream would end the
|
|
700
|
+
// supervisor silently -- the exact class of death this file exists to make impossible.
|
|
701
|
+
if (pid === process.pid) {
|
|
702
|
+
say(`[w2m-supervisor] refusing to terminate pid=${pid}: that is this supervisor`);
|
|
703
|
+
return false;
|
|
704
|
+
}
|
|
705
|
+
if (process.platform !== 'win32') {
|
|
706
|
+
try {
|
|
707
|
+
process.kill(pid, 'SIGKILL');
|
|
708
|
+
} catch {
|
|
709
|
+
/* already gone */
|
|
710
|
+
}
|
|
711
|
+
return !isAlive(pid);
|
|
712
|
+
}
|
|
713
|
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
714
|
+
if (!isAlive(pid)) return true;
|
|
715
|
+
for (const flags of [['/T', '/F'], ['/F']]) {
|
|
716
|
+
spawnSync('taskkill', ['/PID', String(pid), ...flags], { encoding: 'utf8', windowsHide: true, timeout: timeoutMs });
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
return !isAlive(pid);
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
// ---------------------------------------------------------------------------------------------
|
|
723
|
+
// ITEM 2: REAP BY IDENTITY, AT STARTUP
|
|
241
724
|
//
|
|
242
725
|
// Detaching the agent is what stops a console teardown from taking the machine off the fleet, and it
|
|
243
726
|
// has a price that was measured: every supervisor generation that dies without stopping its child
|
|
244
|
-
// leaves an orphan behind
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
// is still alive when a new supervisor starts, it is a previous generation's orphan and is terminated
|
|
250
|
-
// before anything else runs. "Trust a graceful exit" is exactly the assumption that failed.
|
|
727
|
+
// leaves an orphan behind, and a five-minute watchdog makes a new generation per tick. The reap
|
|
728
|
+
// below is the payment: it looks at every agent process of this identity and keeps none of them,
|
|
729
|
+
// because at this point in the file this generation has the lock and has not spawned anything, so
|
|
730
|
+
// **any** agent process alive right now belongs to a dead generation. "Trust a graceful exit" is the
|
|
731
|
+
// assumption that failed; "trust the process table" is the one that holds.
|
|
251
732
|
// ---------------------------------------------------------------------------------------------
|
|
733
|
+
let startupReapNote = 'not attempted';
|
|
252
734
|
try {
|
|
253
|
-
const
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
735
|
+
const stale = agentProcesses();
|
|
736
|
+
if (stale === null) {
|
|
737
|
+
startupReapNote = 'could not enumerate the process table; nothing reaped';
|
|
738
|
+
say(`[w2m-supervisor] ${startupReapNote}`);
|
|
739
|
+
} else {
|
|
740
|
+
const killedPids = [];
|
|
741
|
+
const survivors = [];
|
|
742
|
+
for (const row of stale) {
|
|
743
|
+
if (row.pid === currentChildPid) continue; // impossible before the first spawn, but never kill our own child
|
|
744
|
+
const killed = terminate(row.pid);
|
|
745
|
+
if (killed) killedPids.push(row.pid);
|
|
746
|
+
else survivors.push(row.pid);
|
|
747
|
+
say(
|
|
748
|
+
`[w2m-supervisor] reaped orphan agent pid=${row.pid}${killed ? '' : " (would not die; it may belong to another user)"}` +
|
|
749
|
+
` cmd=${row.cmd.slice(0, 200)}`,
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
orphansReaped = killedPids.length;
|
|
753
|
+
orphansReapedPids = killedPids;
|
|
754
|
+
startupReapNote =
|
|
755
|
+
stale.length === 0
|
|
756
|
+
? 'no orphan agents found'
|
|
757
|
+
: `reaped ${killedPids.length} of ${stale.length}${survivors.length > 0 ? ` (survived: pids=[${survivors.join(',')}])` : ''}`;
|
|
758
|
+
say(
|
|
759
|
+
`[w2m-supervisor] start-time reap by identity (state=${state}): ${startupReapNote}` +
|
|
760
|
+
`${killedPids.length > 0 ? ` pids=[${killedPids.join(',')}]` : ''}`,
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
} catch (error) {
|
|
764
|
+
startupReapNote = `reap failed: ${error.message}`;
|
|
765
|
+
say(`[w2m-supervisor] ${startupReapNote}`);
|
|
766
|
+
}
|
|
767
|
+
// The heartbeat is written *before* the first spawn so the reap and the port decision are visible
|
|
768
|
+
// even if this generation dies in the next second -- a fact in a file outlives a process.
|
|
769
|
+
const startupSuperseded = (() => {
|
|
770
|
+
try {
|
|
771
|
+
const others = supervisorProcesses();
|
|
772
|
+
return others === null ? null : others.length;
|
|
773
|
+
} catch {
|
|
774
|
+
return null;
|
|
775
|
+
}
|
|
776
|
+
})();
|
|
777
|
+
if (startupSuperseded !== null && startupSuperseded > 0) {
|
|
778
|
+
say(
|
|
779
|
+
`[w2m-supervisor] note: ${startupSuperseded} other supervisor process(es) of this identity are alive; ` +
|
|
780
|
+
`the lock -- not a kill -- is what stops them from supervising anything. Reported, not killed: ` +
|
|
781
|
+
`ending a live supervisor here would race the agent it may be about to own.`,
|
|
782
|
+
);
|
|
783
|
+
}
|
|
784
|
+
trace('startup-reap', { orphans_reaped: orphansReaped, pids: orphansReapedPids, note: startupReapNote });
|
|
785
|
+
writeHeartbeat({ state: 'backoff' });
|
|
786
|
+
|
|
787
|
+
// ---------------------------------------------------------------------------------------------
|
|
788
|
+
// ITEM 3: A HELD PUNCH PORT IS A FATAL STARTUP ERROR, NOT A QUIET DEGRADATION
|
|
789
|
+
//
|
|
790
|
+
// The agent's own design is deliberate and correct as library behaviour: `startP2P()` records
|
|
791
|
+
// `P2P_BIND_FAILED` and returns `{ok:false}` because a machine whose fast path is broken is still a
|
|
792
|
+
// working machine over the relay, and the agent's comment says so: "never fatal". The fleet measured
|
|
793
|
+
// what that costs when the port is held by an *orphan of the same identity*: nine agents ran, eight
|
|
794
|
+
// of them with no direct path, every dispatch fell back to the relay with `P2P_UNREACHABLE`, and
|
|
795
|
+
// because a degraded agent is indistinguishable from a healthy one from outside -- same log line
|
|
796
|
+
// volume, same stream, `state: running` -- nothing on this machine or on the fleet noticed for
|
|
797
|
+
// fourteen hours.
|
|
798
|
+
//
|
|
799
|
+
// So the choice between (a) fatal and (b) degraded-but-visible is made here as (a), for these
|
|
800
|
+
// reasons:
|
|
801
|
+
// * The supervisor can decide it *before* spawning, which is the only place the information is
|
|
802
|
+
// clean: `probePort()` binds the punch port itself. If the bind succeeds, the port is free and
|
|
803
|
+
// the agent's own bind cannot fail for the one reason that has ever been observed here; if it
|
|
804
|
+
// fails, something holds the port and the agent's bind will fail in exactly the same way.
|
|
805
|
+
// * (b) cannot be implemented honestly from out here. The supervisor cannot see the agent's
|
|
806
|
+
// in-process `p2pStartResult`, and parsing the agent's log for `P2P_BIND_FAILED` makes a log
|
|
807
|
+
// line load-bearing -- the file this whole design exists because a log line is not guaranteed
|
|
808
|
+
// to be written. A heartbeat that says "degraded" only when a log happened to be flushed is a
|
|
809
|
+
// heartbeat that lies during the incident it is for.
|
|
810
|
+
// * Retrying is what an operator would do, and it is safe to automate *because of the lock*: with
|
|
811
|
+
// one generation possible, a retry cannot produce a second agent, so "reap and retry" costs one
|
|
812
|
+
// backoff and removes the accumulated state rather than compounding it.
|
|
813
|
+
// The state is not hidden: the heartbeat carries `port.state` (free/bound/bound-by-other/held) and a
|
|
814
|
+
// named `last_reason` (P2P_PORT_HELD_BY_OTHER / P2P_PORT_NOT_BOUND), so "the agent is up but has no
|
|
815
|
+
// punch port" can never again be a silent condition -- it is either repaired or said out loud.
|
|
816
|
+
// ---------------------------------------------------------------------------------------------
|
|
817
|
+
|
|
818
|
+
/**
|
|
819
|
+
* Can this process bind the punch port? The probe is the agent's own setup (`udp4`, `0.0.0.0`), so
|
|
820
|
+
* its answer is the agent's answer. A successful bind is closed immediately: it is a question, not
|
|
821
|
+
* a claim on the port.
|
|
822
|
+
*
|
|
823
|
+
* @returns {Promise<{free: boolean, code: string|null}>}
|
|
824
|
+
*/
|
|
825
|
+
function probePort(port) {
|
|
826
|
+
return new Promise((resolve) => {
|
|
827
|
+
let settled = false;
|
|
828
|
+
const done = (result) => {
|
|
829
|
+
if (settled) return;
|
|
830
|
+
settled = true;
|
|
831
|
+
resolve(result);
|
|
832
|
+
};
|
|
833
|
+
let socket;
|
|
257
834
|
try {
|
|
258
|
-
|
|
259
|
-
} catch {
|
|
260
|
-
|
|
835
|
+
socket = createSocket('udp4');
|
|
836
|
+
} catch (error) {
|
|
837
|
+
done({ free: false, code: error.code ?? error.message });
|
|
838
|
+
return;
|
|
261
839
|
}
|
|
262
|
-
|
|
840
|
+
const timer = setTimeout(() => {
|
|
841
|
+
try {
|
|
842
|
+
socket.close();
|
|
843
|
+
} catch {
|
|
844
|
+
/* never bound */
|
|
845
|
+
}
|
|
846
|
+
done({ free: false, code: 'ETIMEDOUT' });
|
|
847
|
+
}, 3_000);
|
|
848
|
+
timer.unref?.();
|
|
849
|
+
socket.on('error', (error) => {
|
|
850
|
+
clearTimeout(timer);
|
|
851
|
+
done({ free: false, code: error.code ?? error.message });
|
|
852
|
+
});
|
|
853
|
+
socket.bind({ address: '0.0.0.0', port }, () => {
|
|
854
|
+
clearTimeout(timer);
|
|
263
855
|
try {
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
856
|
+
socket.close();
|
|
857
|
+
} catch {
|
|
858
|
+
/* closing a bound socket cannot fail meaningfully */
|
|
859
|
+
}
|
|
860
|
+
done({ free: true, code: null });
|
|
861
|
+
});
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
function notePort(state_, ownerPid, note) {
|
|
866
|
+
portInfo.state = state_;
|
|
867
|
+
portInfo.owner_pid = ownerPid;
|
|
868
|
+
portInfo.note = note;
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
/**
|
|
872
|
+
* The pid holding the port, when the OS will say. Diagnostic only: absence is not a failure.
|
|
873
|
+
*
|
|
874
|
+
* `netstat -ano` rather than `Get-NetUDPEndpoint`: measured on this machine, one PowerShell start is
|
|
875
|
+
* ~2.6 s against ~0.09 s for netstat, and this runs inside the startup path where a slow answer is
|
|
876
|
+
* indistinguishable from a hung agent. The cmdlet is the fallback for a host without netstat, and
|
|
877
|
+
* `withName` asks for the process name too -- worth 2.6 s only when a human has to read the answer,
|
|
878
|
+
* which is exactly when a foreign process holds the port.
|
|
879
|
+
*/
|
|
880
|
+
function portOwner(withName = false) {
|
|
881
|
+
if (process.platform !== 'win32') return null;
|
|
882
|
+
const fromNetstat = () => {
|
|
883
|
+
const result = spawnSync('netstat', ['-ano', '-p', 'UDP'], { encoding: 'utf8', windowsHide: true, timeout: 10_000 });
|
|
884
|
+
if (result.error || typeof result.stdout !== 'string') return null;
|
|
885
|
+
for (const line of result.stdout.split(/\r?\n/)) {
|
|
886
|
+
const match = /^\s*UDP\s+\S*:(\d+)\s+\S+\s+(\d+)\s*$/.exec(line);
|
|
887
|
+
if (match && Number(match[1]) === p2pPort) return { pid: Number(match[2]), name: null };
|
|
888
|
+
}
|
|
889
|
+
return null;
|
|
890
|
+
};
|
|
891
|
+
try {
|
|
892
|
+
const quick = fromNetstat();
|
|
893
|
+
if (quick !== null && !withName) return quick;
|
|
894
|
+
// Ask the cmdlet once: it names the process, and it is the only route when netstat is absent or
|
|
895
|
+
// when its output has no line for this port at all.
|
|
896
|
+
const script = `$e=Get-NetUDPEndpoint -LocalPort ${p2pPort} -ErrorAction SilentlyContinue | Select-Object -First 1; if ($e) { $p=Get-Process -Id $e.OwningProcess -ErrorAction SilentlyContinue; "$($e.OwningProcess) $($p.ProcessName)" }`;
|
|
897
|
+
const result = spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], {
|
|
898
|
+
encoding: 'utf8',
|
|
899
|
+
windowsHide: true,
|
|
900
|
+
timeout: 10_000,
|
|
901
|
+
});
|
|
902
|
+
const text = typeof result.stdout === 'string' ? result.stdout.trim() : '';
|
|
903
|
+
const [pidText, name] = text.split(/\s+/);
|
|
904
|
+
const pid = Number(pidText);
|
|
905
|
+
if (Number.isInteger(pid) && pid > 0) return { pid, name: name ?? 'unknown' };
|
|
906
|
+
return quick;
|
|
907
|
+
} catch {
|
|
908
|
+
return null;
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
const killedByUs = (pid) => typeof pid === 'number' && pid === currentChildPid;
|
|
913
|
+
|
|
914
|
+
/**
|
|
915
|
+
* Wait for the punch port to be free, reaping every other agent of this identity while waiting.
|
|
916
|
+
*
|
|
917
|
+
* UDP has no TIME_WAIT, so a killed holder releases the port as soon as its process is gone; the
|
|
918
|
+
* wait is for `taskkill` to finish, not for a timer. The one case this cannot repair is a foreign
|
|
919
|
+
* process on the same port, and that is reported by name rather than swallowed.
|
|
920
|
+
*
|
|
921
|
+
* @returns {Promise<{free: boolean, reason: string|null, heldBy: object|null}>}
|
|
922
|
+
*/
|
|
923
|
+
async function ensurePortFree() {
|
|
924
|
+
trace('port-gate', { port: p2pPort, mode: p2pMode });
|
|
925
|
+
if (p2pPort === null || p2pMode === 'relay') {
|
|
926
|
+
notePort('not-configured', null, p2pPort === null ? 'no pinned punch port; the agent picks an ephemeral one' : 'p2p mode is relay');
|
|
927
|
+
return { free: true, reason: null, heldBy: null };
|
|
928
|
+
}
|
|
929
|
+
portInfo.configured = p2pPort;
|
|
930
|
+
let probe = await probePort(p2pPort);
|
|
931
|
+
trace('port-gate-probe', { free: probe.free, code: probe.code });
|
|
932
|
+
if (probe.free) {
|
|
933
|
+
notePort('free', null, 'the punch port was free before the spawn');
|
|
934
|
+
return { free: true, reason: null, heldBy: null };
|
|
935
|
+
}
|
|
936
|
+
// Something holds it. If it is an agent of this identity, it is an orphan that the start-time reap
|
|
937
|
+
// should already have taken; a second pass covers the race where it appeared afterwards.
|
|
938
|
+
const holder = portOwner();
|
|
939
|
+
const agents = agentProcesses();
|
|
940
|
+
const agentPids = agents === null ? [] : agents.map((row) => row.pid);
|
|
941
|
+
const extra = agentPids.filter((pid) => !killedByUs(pid));
|
|
942
|
+
if (extra.length > 0) {
|
|
943
|
+
say(`[w2m-supervisor] punch port ${p2pPort} is held and ${extra.length} agent process(es) of this identity are alive: pids=[${extra.join(',')}] -- reaping and retrying`);
|
|
944
|
+
const killedNow = [];
|
|
945
|
+
for (const pid of extra) {
|
|
946
|
+
if (terminate(pid)) killedNow.push(pid);
|
|
947
|
+
}
|
|
948
|
+
orphansReaped += killedNow.length;
|
|
949
|
+
orphansReapedPids = [...new Set(orphansReapedPids.concat(killedNow))];
|
|
950
|
+
}
|
|
951
|
+
for (let attempt = 0; attempt < 15; attempt += 1) {
|
|
952
|
+
await new Promise((resolve) => {
|
|
953
|
+
setTimeout(resolve, 200);
|
|
954
|
+
});
|
|
955
|
+
probe = await probePort(p2pPort);
|
|
956
|
+
trace('port-gate-retry', { attempt, free: probe.free, code: probe.code });
|
|
957
|
+
if (probe.free) {
|
|
958
|
+
notePort('free', null, `the punch port came free after the reap${extra.length > 0 ? ` of pids=[${extra.join(',')}]` : ''}`);
|
|
959
|
+
return { free: true, reason: null, heldBy: null };
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
// Still held after every agent of this identity was taken out: it is somebody else's socket. No
|
|
963
|
+
// amount of retrying will change that, and starting the agent anyway is the fourteen-hour failure.
|
|
964
|
+
const owner = holder ?? portOwner(true);
|
|
965
|
+
notePort('bound-by-other', owner?.pid ?? null, `the punch port is held by ${owner ? `pid=${owner.pid} (${owner.name})` : 'another process'}, which is not an agent of this identity`);
|
|
966
|
+
return {
|
|
967
|
+
free: false,
|
|
968
|
+
reason: 'P2P_PORT_HELD_BY_OTHER',
|
|
969
|
+
heldBy: owner,
|
|
970
|
+
};
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
/** Give up on the port and let the named reason reach the heartbeat before the next backoff. */
|
|
974
|
+
async function refuseToSpawn(reason, heldBy) {
|
|
975
|
+
lastReason = reason;
|
|
976
|
+
const detail =
|
|
977
|
+
reason === 'P2P_PORT_HELD_BY_OTHER'
|
|
978
|
+
? `punch port ${p2pPort} is held by ${heldBy ? `pid=${heldBy.pid} (${heldBy.name})` : 'an unidentified process'} and it is not an agent of this identity`
|
|
979
|
+
: `the agent did not bind punch port ${p2pPort} within the deadline`;
|
|
980
|
+
say(`[w2m-supervisor] refusing to start: ${reason}: ${detail} -- the agent would run with no direct path and every dispatch would fall back to the relay`);
|
|
981
|
+
if (portInfo.state !== 'bound-by-other') notePort('held', heldBy?.pid ?? null, detail);
|
|
982
|
+
writeHeartbeat({ state: 'degraded', last_reason: reason, child_pid: null, child_started_at: null });
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
/**
|
|
986
|
+
* Did the agent actually take the punch port?
|
|
987
|
+
*
|
|
988
|
+
* `probePort` answers "is the port free": after the spawn, free means the agent has not bound it. As
|
|
989
|
+
* the agent's own bind is not observable from here (it is in-process), this is the honest check the
|
|
990
|
+
* supervisor can make, and it turns the one silent degradation that has actually happened into a
|
|
991
|
+
* named restart: if the port is still free after the deadline, the agent is running without a direct
|
|
992
|
+
* path, so it is stopped and reported rather than left to look healthy.
|
|
993
|
+
*
|
|
994
|
+
* @returns {Promise<{bound: boolean, code: string|null}>}
|
|
995
|
+
*/
|
|
996
|
+
async function waitForPortBound(childPid, deadlineMs) {
|
|
997
|
+
const began = Date.now();
|
|
998
|
+
for (;;) {
|
|
999
|
+
const probe = await probePort(p2pPort);
|
|
1000
|
+
trace('port-after-spawn-probe', { pid: childPid, free: probe.free, code: probe.code });
|
|
1001
|
+
if (!probe.free) {
|
|
1002
|
+
const owner = portOwner();
|
|
1003
|
+
trace('port-after-spawn-owner', { owner: owner?.pid ?? null, name: owner?.name ?? null });
|
|
1004
|
+
if (owner === null) {
|
|
1005
|
+
// Bound, but the OS would not name the holder. The bind succeeded for us one moment ago and
|
|
1006
|
+
// the agent is alive, so the honest reading is "the agent took it"; `portOwner` is a
|
|
1007
|
+
// diagnostic, and its absence must not turn a working agent into a restart loop.
|
|
1008
|
+
notePort('bound', childPid, `punch port ${p2pPort} is bound and the agent (pid=${childPid}) is alive; the OS did not name the holder`);
|
|
1009
|
+
return { bound: true, code: null };
|
|
271
1010
|
}
|
|
1011
|
+
if (owner.pid === childPid) {
|
|
1012
|
+
notePort('bound', owner.pid, `the punch port is bound by the agent (pid=${owner.pid})`);
|
|
1013
|
+
return { bound: true, code: null };
|
|
1014
|
+
}
|
|
1015
|
+
notePort('bound-by-other', owner.pid, `the punch port is bound by pid=${owner.pid} (${owner.name}), not by the agent pid=${childPid}`);
|
|
1016
|
+
return { bound: false, code: 'P2P_PORT_HELD_BY_OTHER' };
|
|
272
1017
|
}
|
|
1018
|
+
if (!isAlive(childPid) || Date.now() - began >= deadlineMs) return { bound: false, code: probe.code };
|
|
1019
|
+
await new Promise((resolve) => {
|
|
1020
|
+
setTimeout(resolve, 500);
|
|
1021
|
+
});
|
|
273
1022
|
}
|
|
274
|
-
} catch {
|
|
275
|
-
// No heartbeat yet, or unreadable: nothing to reap, and that is not an error worth stopping for.
|
|
276
1023
|
}
|
|
1024
|
+
|
|
1025
|
+
async function stopChild(child, why) {
|
|
1026
|
+
if (!child || child.pid === undefined) return;
|
|
1027
|
+
if (!isAlive(child.pid)) return;
|
|
1028
|
+
say(`[w2m-supervisor] stopping agent pid=${child.pid} (${why})`);
|
|
1029
|
+
terminate(child.pid);
|
|
1030
|
+
}
|
|
1031
|
+
|
|
277
1032
|
// A periodic beat, so "the supervisor itself is alive but a worker is wedged" is visible too.
|
|
278
|
-
const beat = setInterval(() =>
|
|
1033
|
+
const beat = setInterval(() => {
|
|
1034
|
+
trace('beat', { port_state: portInfo.state, child: currentChildPid });
|
|
1035
|
+
writeHeartbeat({});
|
|
1036
|
+
}, 30_000);
|
|
279
1037
|
beat.unref();
|
|
280
1038
|
|
|
1039
|
+
const portWaitMs = Number.isFinite(Number(pick('port-wait-ms', 30_000))) ? Number(pick('port-wait-ms', 30_000)) : 30_000;
|
|
1040
|
+
|
|
281
1041
|
while (!stopping) {
|
|
282
1042
|
runs += 1;
|
|
283
1043
|
const startedAt = Date.now();
|
|
1044
|
+
trace('loop-iteration', { runs, stopping, currentChildPid });
|
|
1045
|
+
// Item 3, first half: never spawn into a port that is already taken. This is where an orphan from
|
|
1046
|
+
// a previous generation is discovered even if it appeared after the start-time reap.
|
|
1047
|
+
const gate = await ensurePortFree();
|
|
1048
|
+
if (stopping) break;
|
|
1049
|
+
if (!gate.free) {
|
|
1050
|
+
await refuseToSpawn(gate.reason, gate.heldBy);
|
|
1051
|
+
exitTimes.push(Date.now());
|
|
1052
|
+
const record = writeHeartbeat({});
|
|
1053
|
+
const wait = record.state === 'degraded' ? degradedBackoffMs : backoffMs;
|
|
1054
|
+
// No hot loop here: the port gate is paced by the same backoff a crashed agent gets, so a
|
|
1055
|
+
// persistently occupied port produces a slow, visible, well-labelled heartbeat rather than a
|
|
1056
|
+
// spawn storm.
|
|
1057
|
+
say(`[w2m-supervisor] run ${runs} refused to start (${gate.reason}) -- waiting ${Math.round(wait / 1000)}s`);
|
|
1058
|
+
await new Promise((resolve) => {
|
|
1059
|
+
setTimeout(resolve, wait);
|
|
1060
|
+
});
|
|
1061
|
+
continue;
|
|
1062
|
+
}
|
|
284
1063
|
// ---------------------------------------------------------------------------------------------
|
|
285
1064
|
// THE AGENT MUST NOT SHARE THIS PROCESS'S CONSOLE.
|
|
286
1065
|
//
|
|
@@ -309,15 +1088,63 @@ while (!stopping) {
|
|
|
309
1088
|
stdio: ['ignore', logFd, logFd],
|
|
310
1089
|
windowsHide: true,
|
|
311
1090
|
});
|
|
312
|
-
|
|
1091
|
+
currentChildPid = child.pid ?? null;
|
|
1092
|
+
currentChildStartedAt = stamp();
|
|
1093
|
+
lastReason = null;
|
|
1094
|
+
if (child.pid === undefined) {
|
|
1095
|
+
// No pid at all: there is nothing to own and nothing to check, so this is one named run rather
|
|
1096
|
+
// than a spawn followed by a 30 s port wait that could only ever conclude "not bound".
|
|
1097
|
+
exitTimes.push(Date.now());
|
|
1098
|
+
const failed = await exited;
|
|
1099
|
+
say(`[w2m-supervisor] run ${runs} never started (spawn returned no pid); exit code=${failed.exitCode} -- restarting`);
|
|
1100
|
+
const record = writeHeartbeat({ last_exit_code: failed.exitCode, last_exit_at: stamp(), last_reason: 'SPAWN_FAILED', child_pid: null, child_started_at: null });
|
|
1101
|
+
await new Promise((resolve) => {
|
|
1102
|
+
setTimeout(resolve, record.state === 'degraded' ? degradedBackoffMs : backoffMs);
|
|
1103
|
+
});
|
|
1104
|
+
continue;
|
|
1105
|
+
}
|
|
1106
|
+
writeHeartbeat({ state: 'running', child_pid: currentChildPid, child_started_at: currentChildStartedAt });
|
|
1107
|
+
say(`[w2m-supervisor] run ${runs} started agent pid=${currentChildPid}`);
|
|
1108
|
+
trace('spawned', { pid: currentChildPid });
|
|
313
1109
|
|
|
314
|
-
|
|
1110
|
+
// Item 3, second half: confirm the agent really took the port, and stop it if it did not. Two
|
|
1111
|
+
// checks run concurrently, because either is a reason to stop: the port deadline, and the child's
|
|
1112
|
+
// own exit (the loop below must not hold a dead child's heartbeat open as `running`).
|
|
1113
|
+
const exited = new Promise((resolve) => {
|
|
315
1114
|
child.on('exit', (exitCode, signal) => resolve({ exitCode, signal }));
|
|
316
1115
|
child.on('error', (error) => {
|
|
317
1116
|
say(`[w2m-supervisor] spawn failed: ${error.message}`);
|
|
318
1117
|
resolve({ exitCode: null, signal: null });
|
|
319
1118
|
});
|
|
320
1119
|
});
|
|
1120
|
+
const portCheck =
|
|
1121
|
+
p2pPort === null || p2pMode === 'relay'
|
|
1122
|
+
? Promise.resolve({ bound: true, code: null })
|
|
1123
|
+
: waitForPortBound(child.pid, portWaitMs);
|
|
1124
|
+
const winner = await Promise.race([
|
|
1125
|
+
portCheck.then((result) => ({ kind: 'port', result })),
|
|
1126
|
+
exited.then((result) => ({ kind: 'exit', result })),
|
|
1127
|
+
]).catch((error) => {
|
|
1128
|
+
// A rejection here would otherwise be the one death this file cannot describe: Node ends the
|
|
1129
|
+
// process on an unhandled rejection, so the supervisor would disappear with nothing but a stack
|
|
1130
|
+
// trace. Named, logged, and turned into a decision instead.
|
|
1131
|
+
say(`[w2m-supervisor] the punch-port check failed: ${error.stack ?? error.message}`);
|
|
1132
|
+
trace('port-check-threw', { message: error.message });
|
|
1133
|
+
return { kind: 'exit', result: { exitCode: null, signal: null } };
|
|
1134
|
+
});
|
|
1135
|
+
if (winner.kind === 'port' && !winner.result.bound && isAlive(child.pid)) {
|
|
1136
|
+
// The port went to somebody else between the gate and the bind: the agent is alive but had no
|
|
1137
|
+
// direct path, which is the exact condition that used to run silently for fourteen hours.
|
|
1138
|
+
const reason = winner.result.code === 'P2P_PORT_HELD_BY_OTHER' ? 'P2P_PORT_HELD_BY_OTHER' : 'P2P_PORT_NOT_BOUND';
|
|
1139
|
+
if (reason === 'P2P_PORT_NOT_BOUND') {
|
|
1140
|
+
notePort('held', portOwner()?.pid ?? null, `nothing bound punch port ${p2pPort} within ${Math.round(portWaitMs / 1000)}s of the spawn`);
|
|
1141
|
+
}
|
|
1142
|
+
await refuseToSpawn(reason, portInfo.owner_pid === null ? null : { pid: portInfo.owner_pid, name: 'unknown' });
|
|
1143
|
+
await stopChild(child, `${reason}: it had no punch port, so every dispatch would use the relay`);
|
|
1144
|
+
}
|
|
1145
|
+
const result = await exited;
|
|
1146
|
+
if (currentChildPid === child.pid) currentChildPid = null;
|
|
1147
|
+
currentChildStartedAt = null;
|
|
321
1148
|
exitTimes.push(Date.now());
|
|
322
1149
|
|
|
323
1150
|
const lived = Math.round((Date.now() - startedAt) / 1000);
|
|
@@ -344,4 +1171,4 @@ while (!stopping) {
|
|
|
344
1171
|
});
|
|
345
1172
|
}
|
|
346
1173
|
writeHeartbeat({ state: 'stopped' });
|
|
347
|
-
|
|
1174
|
+
dropLock('this generation is exiting');
|