@twinsearth/w2m-dsh-plugin 0.5.9 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -33,9 +33,34 @@
33
33
  // (cmd, PowerShell 5.1, Task Scheduler). A config file has no quoting rules at all, it is one more
34
34
  // thing the installer can write atomically, and it stays readable for whoever debugs the service.
35
35
  //
36
+ // ONE SUPERVISOR, ONE AGENT -- WHY THE LOCK EXISTS
37
+ //
38
+ // Measured on this machine, over one afternoon: the watchdog trigger restarted the supervisor every
39
+ // five minutes while the previous generation's *detached* agent was still alive, and the start-time
40
+ // reap below could only ever see the single pid the heartbeat remembered. Fourteen `w2m` processes
41
+ // accumulated, nine of them `w2m-localside` agents sharing one identity. The punch port (41235) was
42
+ // held by an agent started thirteen hours earlier, so every later agent logged
43
+ // `P2P_BIND_FAILED: bind EADDRINUSE 0.0.0.0:41235`, the relay saw `streams: 9`, and every dispatch
44
+ // fell back to the relay with `P2P_UNREACHABLE`. The machine was on the fleet and useless for
45
+ // fourteen hours, and nothing in the heartbeat said so.
46
+ //
47
+ // The lock at the top of this file is what makes "one supervisor, one agent" true *by construction*
48
+ // rather than by remembering to clean up: a second supervisor generation cannot start work at all.
36
49
  // Nothing here is Windows-specific except the defaults; the same supervisor works on POSIX hosts.
37
- import { spawn } from 'node:child_process';
38
- import { createWriteStream, existsSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
50
+ import { spawn, spawnSync } from 'node:child_process';
51
+ import { createSocket } from 'node:dgram';
52
+ import {
53
+ appendFileSync,
54
+ closeSync,
55
+ existsSync,
56
+ mkdirSync,
57
+ openSync,
58
+ readFileSync,
59
+ renameSync,
60
+ rmSync,
61
+ writeFileSync,
62
+ writeSync,
63
+ } from 'node:fs';
39
64
  import { dirname, join } from 'node:path';
40
65
 
41
66
  /** Minimal `--flag value` / `--flag` parser: no dependency, no surprises. */
@@ -147,17 +172,253 @@ if (!existsSync(bin)) {
147
172
  process.exit(78); // EX_CONFIG: a service manager should not keep retrying this
148
173
  }
149
174
 
150
- mkdirSync(dirname(logPath), { recursive: true });
151
- const log = createWriteStream(logPath, { flags: 'a' });
175
+ // Where this generation's lock, heartbeat and sidecar state live. The log's directory is the run
176
+ // directory: everything the installer wrote for one service sits together, and two services with
177
+ // different logs are two unrelated installations that must not share a lock.
178
+ const PROC_MARKER_AGENT = 'w2m-localside';
179
+ const PROC_MARKER_SUPERVISOR = 'w2m-agent-supervisor';
180
+
181
+ /**
182
+ * Does this command line belong to a supervisor of THIS installation?
183
+ *
184
+ * Used for two decisions that must agree with each other: whether a pid recorded in the lock is still
185
+ * a supervisor (the lock may be taken over otherwise), and whether a process of this identity is an
186
+ * agent to reap or a supervisor to leave alone. A supervisor's command line carries the agent stub's
187
+ * path -- `--bin ...\w2m-localside-stub.mjs` -- so "mentions w2m-localside" is true of a supervisor,
188
+ * and a reap that matched on identity alone would kill its own program. Measured: it did.
189
+ */
190
+ function isSupervisorCommandLine(cmd) {
191
+ return (
192
+ typeof cmd === 'string' &&
193
+ cmd.includes(PROC_MARKER_SUPERVISOR) &&
194
+ cmd.includes('--config') &&
195
+ (configPath === null || cmd.includes(configPath))
196
+ );
197
+ }
198
+ const runDir = dirname(logPath);
199
+ mkdirSync(runDir, { recursive: true });
152
200
  // A second handle on the same file, as a descriptor: this is what the detached agent writes into,
153
201
  // because a descriptor is not connected to this process's console or to any pipe of ours.
154
202
  const logFd = openSync(logPath, 'a');
155
203
  const stamp = () => new Date().toISOString();
204
+ /**
205
+ * One line, two destinations, both synchronous, and both for a measured reason.
206
+ *
207
+ * The log is written with `appendFileSync` rather than through a `WriteStream`: a stream buffers, and
208
+ * the generation that most needs to explain itself is the one that exits immediately -- the watchdog
209
+ * tick that finds the lock held prints its no-op line and exits, while a buffered write is still in
210
+ * memory and is discarded with the process. Measured: the line was missing from the log, so the log
211
+ * said nothing about the most common thing this file now does.
212
+ */
156
213
  const say = (line) => {
157
214
  console.log(line);
158
- log.write(`${line}\n`);
215
+ try {
216
+ appendFileSync(logPath, `${line}\n`);
217
+ } catch {
218
+ /* the console line is still out; a log that cannot be written must not stop the supervisor */
219
+ }
220
+ };
221
+ /**
222
+ * An opt-in trace of the decisions that are otherwise only visible as their consequences, written
223
+ * as a JSON line per event. `verify-supervisor.mjs` sets `W2M_SUPERVISOR_TRACE` when it needs to
224
+ * explain a failure; nothing in a normal service run sets it, and an unset variable costs one
225
+ * comparison.
226
+ */
227
+ const tracePath = typeof process.env.W2M_SUPERVISOR_TRACE === 'string' && process.env.W2M_SUPERVISOR_TRACE !== ''
228
+ ? process.env.W2M_SUPERVISOR_TRACE
229
+ : null;
230
+ const traceFd = tracePath === null ? null : openSync(tracePath, 'a');
231
+ function trace(event, detail) {
232
+ if (traceFd === null) return;
233
+ try {
234
+ writeSync(traceFd, `${JSON.stringify({ at: stamp(), event, ...detail })}\n`);
235
+ } catch {
236
+ /* a trace that cannot be written must never be a reason to stop supervising */
237
+ }
238
+ }
239
+ // The last line of defence: a process that vanishes without a word is the exact failure this file
240
+ // exists to make impossible, so nothing is allowed to end it silently.
241
+ process.on('uncaughtException', (error) => {
242
+ trace('uncaught-exception', { message: error?.message ?? String(error), stack: error?.stack ?? null });
243
+ say(`[w2m-supervisor] uncaught exception: ${error?.stack ?? error?.message ?? String(error)}`);
244
+ });
245
+ process.on('unhandledRejection', (reason) => {
246
+ trace('unhandled-rejection', { message: reason?.message ?? String(reason), stack: reason?.stack ?? null });
247
+ say(`[w2m-supervisor] unhandled rejection: ${reason?.stack ?? reason?.message ?? String(reason)}`);
248
+ });
249
+
250
+ // ---------------------------------------------------------------------------------------------
251
+ // THE SINGLE-INSTANCE LOCK
252
+ //
253
+ // The watchdog trigger in `install-service.ps1` exists because a console-teardown death cannot be
254
+ // logged from inside the process it kills: the task must be able to say "be running, or start".
255
+ // That trigger is only safe if starting a *second* generation while a healthy one runs is a no-op,
256
+ // and that is exactly what the lock buys. Without it the trigger is a process factory: every
257
+ // restart of a supervisor whose agent survived leaves another agent behind, and the old reap --
258
+ // which trusted the single `child_pid` in the heartbeat -- could never catch up.
259
+ //
260
+ // `wx` is the atomic part: `open(..., 'wx')` is O_CREAT|O_EXCL, so of two generations racing to
261
+ // start, exactly one creates the file and the other gets EEXIST. The file *is* the lock only until
262
+ // its holder dies, though, and this project has measured two ways for that to happen abruptly:
263
+ // `Stop-Process -Force`, and a console teardown that kills the tree with 0xC000013A. Neither runs
264
+ // any handler here, so the file is simply left behind. It is therefore not the existence of the
265
+ // file that decides anything -- it is the liveness of the pid written in it. That is why a stale
266
+ // lock (holder gone, or the file unreadable) is taken over loudly instead of blocking the machine
267
+ // forever, and why `orphans_reaped`/`lock_takeovers` are in the heartbeat: taking a lock over is
268
+ // the normal aftermath of exactly the death this file exists to survive.
269
+ // ---------------------------------------------------------------------------------------------
270
+ const lockPath = typeof pick('lock', null) === 'string' ? pick('lock', null) : join(runDir, 'supervisor.lock');
271
+ /** Takeovers, cumulative across generations: kept beside the lock so a restart does not erase it. */
272
+ const lockStatePath = `${lockPath}.state.json`;
273
+
274
+ function readLockState(file_) {
275
+ try {
276
+ const parsed = JSON.parse(readFileSync(file_, 'utf8').replace(/^\uFEFF/, ''));
277
+ return parsed !== null && typeof parsed === 'object' ? parsed : {};
278
+ } catch {
279
+ return {};
280
+ }
281
+ }
282
+
283
+ /** Best effort, and only used for numbers: an unreadable sidecar is not a reason to refuse to run. */
284
+ function bumpLockTakeovers() {
285
+ try {
286
+ const previous = Number(readLockState(lockStatePath).takeovers ?? 0);
287
+ const next = Number.isFinite(previous) ? previous + 1 : 1;
288
+ const tmp = `${lockStatePath}.tmp`;
289
+ writeFileSync(tmp, `${JSON.stringify({ takeovers: next, updated_at: stamp() }, null, 2)}\n`, 'utf8');
290
+ renameSync(tmp, lockStatePath);
291
+ return next;
292
+ } catch {
293
+ return null;
294
+ }
295
+ }
296
+
297
+ const takeoversTotal = Number(lockStatePath === '' ? 0 : readLockState(lockStatePath).takeovers ?? 0) || 0;
298
+ const lockInfo = {
299
+ path: lockPath,
300
+ pid: process.pid,
301
+ started_at: stamp(),
302
+ takeovers_total: takeoversTotal,
303
+ takeovers_this_run: 0,
159
304
  };
160
305
 
306
+ /**
307
+ * Is another supervisor generation working right now?
308
+ *
309
+ * Alive is not enough on its own: Windows reuses pids, and a lock left by a killed supervisor can
310
+ * name a pid that now belongs to something else entirely -- long-lived, unrelated, and none of this
311
+ * program's business. Treating that as "held" would wedge the machine until the process happened to
312
+ * exit, which is the failure this whole file is against. So the pid is checked for being a
313
+ * *supervisor*: node, running this script, with this `--config`. When the process table cannot be
314
+ * read the answer is "assume held", because the one irreversible mistake is a second agent -- the
315
+ * watchdog ticks again in five minutes, and the log line says which of the two happened.
316
+ *
317
+ * @returns {{held: boolean, reason: string}}
318
+ */
319
+ function lockHolder() {
320
+ if (!existsSync(lockPath)) return { held: false, reason: 'lock file is gone (the holder removed it)' };
321
+ let pid = null;
322
+ let holderStartedAt = null;
323
+ let parseNote = 'unreadable';
324
+ try {
325
+ const parsed = JSON.parse(readFileSync(lockPath, 'utf8').replace(/^\uFEFF/, ''));
326
+ // A pid written as a string is still a pid: the field has been written by hand at least once.
327
+ if (Number.isInteger(Number(parsed?.pid)) && Number(parsed.pid) > 0) {
328
+ pid = Number(parsed.pid);
329
+ holderStartedAt = typeof parsed?.started_at === 'string' ? parsed.started_at : null;
330
+ parseNote = 'readable';
331
+ } else {
332
+ parseNote = 'readable but has no usable pid';
333
+ }
334
+ } catch (error) {
335
+ // Unreadable is treated exactly like a dead holder -- see the block comment above: a truncated
336
+ // or half-written lock must not be able to keep the fleet down with a stale claim on the port.
337
+ parseNote = `unreadable (${error.code ?? error.message})`;
338
+ }
339
+ if (pid === null) return { held: false, reason: parseNote };
340
+ if (pid === process.pid) return { held: false, reason: `this process (pid=${pid}) owns it` };
341
+ try {
342
+ process.kill(pid, 0);
343
+ } catch (error) {
344
+ return { held: false, reason: `pid=${pid} is not running (${error.code ?? error.message})` };
345
+ }
346
+ const rows = listProcesses();
347
+ if (rows !== null) {
348
+ const row = rows.find((candidate) => candidate.pid === pid);
349
+ if (row !== undefined && !isSupervisorCommandLine(row.cmd)) {
350
+ return {
351
+ held: false,
352
+ reason: `pid=${pid} is alive but is not a supervisor of this installation (${row.cmd.slice(0, 140)})`,
353
+ };
354
+ }
355
+ }
356
+ return { held: true, reason: `pid=${pid} is a supervisor running this config (lock written ${holderStartedAt ?? 'at an unknown time'}, ${parseNote})` };
357
+ }
358
+
359
+ // The lock is the first thing this process does that can have a side effect, and the last thing it
360
+ // gives up. Everything above this line is argument/config parsing, which changes nothing.
361
+ let lockFd = null;
362
+ for (let attempt = 0; attempt < 3 && lockFd === null; attempt += 1) {
363
+ try {
364
+ lockFd = openSync(lockPath, 'wx');
365
+ } catch (error) {
366
+ if (error.code !== 'EEXIST') {
367
+ // EX_CONFIG, not a crash: no lock means two agents is possible, and running anyway would
368
+ // trade a visible failure for the exact invisible one this file exists to prevent.
369
+ console.error(`w2m-supervisor: cannot create the lock ${lockPath}: ${error.message}`);
370
+ process.exit(78);
371
+ }
372
+ const holder = lockHolder();
373
+ if (holder.held) {
374
+ say(`[w2m-supervisor ${stamp()}] another supervisor generation already holds the lock: ${holder.reason}`);
375
+ say(`[w2m-supervisor ${stamp()}] lock=${lockPath} -- exiting without spawning; the watchdog tick is a no-op`);
376
+ // Deliberately exit 0: a no-op start is the *designed* outcome of a watchdog tick, not a
377
+ // failure, and a non-zero code would make Task Scheduler's RestartOnFailure spin on it.
378
+ process.exit(0);
379
+ }
380
+ try {
381
+ rmSync(lockPath, { force: true });
382
+ say(
383
+ `[w2m-supervisor ${stamp()}] stale lock taken over: not held because ${holder.reason} -- ` +
384
+ `${lockPath} removed, this generation now owns it`,
385
+ );
386
+ const total = bumpLockTakeovers();
387
+ if (Number.isFinite(total)) {
388
+ lockInfo.takeovers_total = total;
389
+ lockInfo.takeovers_this_run += 1;
390
+ }
391
+ } catch (removeError) {
392
+ say(`[w2m-supervisor ${stamp()}] could not clear the stale lock ${lockPath}: ${removeError.message}`);
393
+ process.exit(78);
394
+ }
395
+ }
396
+ }
397
+ if (lockFd === null) {
398
+ say(`[w2m-supervisor ${stamp()}] could not take the lock ${lockPath} after 3 attempts -- exiting without spawning`);
399
+ process.exit(0);
400
+ }
401
+ writeSync(lockFd, `${JSON.stringify({ pid: process.pid, started_at: lockInfo.started_at, argv: process.argv.join(' ') }, null, 2)}\n`);
402
+ // Closed immediately: the lock is the file's existence plus the liveness of the pid inside it, not
403
+ // an open descriptor -- `wx` is what makes the create atomic and that has already happened.
404
+ closeSync(lockFd);
405
+
406
+ let lockHeld = true;
407
+ /** Give the lock up on the way out, so the next generation does not have to call it stale. */
408
+ function dropLock(why) {
409
+ if (!lockHeld) return;
410
+ lockHeld = false;
411
+ try {
412
+ rmSync(lockPath, { force: true });
413
+ say(`[w2m-supervisor ${stamp()}] lock released (${why})`);
414
+ } catch (error) {
415
+ say(`[w2m-supervisor ${stamp()}] could not remove the lock on exit: ${error.message}`);
416
+ }
417
+ }
418
+
419
+ say(`[w2m-supervisor ${stamp()}] supervising ${process.execPath} ${[bin, '--rabbit', rabbit, '--project', project, '--name', name, '--state', state, '--allowed-commands', allowed, ...passthrough].join(' ')}`);
420
+ say(`[w2m-supervisor] lock=${lockPath} holder=pid:${process.pid} takeovers_total=${lockInfo.takeovers_total}`);
421
+
161
422
  const argv = [
162
423
  bin,
163
424
  '--rabbit', rabbit,
@@ -168,7 +429,6 @@ const argv = [
168
429
  ...passthrough,
169
430
  ];
170
431
 
171
- say(`[w2m-supervisor ${stamp()}] supervising ${process.execPath} ${argv.join(' ')}`);
172
432
  let stopping = false;
173
433
  for (const signal of ['SIGINT', 'SIGTERM', 'SIGBREAK']) {
174
434
  process.on(signal, () => {
@@ -179,6 +439,33 @@ for (const signal of ['SIGINT', 'SIGTERM', 'SIGBREAK']) {
179
439
  // Declared before the heartbeat block, which reads it: a `let` referenced earlier in the same scope
180
440
  // would be a temporal-dead-zone throw, not an undefined value.
181
441
  let runs = 0;
442
+ /** The worker that is alive right now, tracked here so every heartbeat can name it. */
443
+ let currentChildPid = null;
444
+ let currentChildStartedAt = null;
445
+ /** Item 2 and item 3 facts, kept for the life of this generation and repeated in every heartbeat. */
446
+ let orphansReaped = 0;
447
+ let orphansReapedPids = [];
448
+ let lastReason = null;
449
+
450
+ // Extra agent flags this supervisor understands for its own bookkeeping. They are still passed to
451
+ // the agent (which ignores unknown `--flags`) and exist so `verify-supervisor.mjs` can drive this
452
+ // file against a stub worker without Windows Task Scheduler and without touching the live agent.
453
+ const reapMatch = typeof pick('reap-match', null) === 'string' ? pick('reap-match', null) : state;
454
+ /**
455
+ * The punch port, taken from the same place the agent takes it. `--p2p-port` arrives through
456
+ * `passthrough` (it is an agent flag, written as `agentArgs: "--p2p-mode auto --p2p-port 41235"` in
457
+ * the installer's config), with the flag/config forms as fallbacks for a direct command line.
458
+ * `0` means "let the OS choose" and is refused by the agent itself, so it is no port to gate on.
459
+ */
460
+ const p2pPort = (() => {
461
+ const at = passthrough.indexOf('--p2p-port');
462
+ const value = at >= 0 ? passthrough[at + 1] : pick('p2p-port', null);
463
+ const parsed = Number(value);
464
+ return Number.isInteger(parsed) && parsed > 0 && parsed <= 65_535 ? parsed : null;
465
+ })();
466
+ const p2pMode = String(pick('p2p-mode', 'auto')).toLowerCase();
467
+ /** Item 3's facts, repeated in every heartbeat for the life of this generation. */
468
+ const portInfo = { configured: p2pPort, state: 'unknown', owner_pid: null, note: null };
182
469
 
183
470
  // ---------------------------------------------------------------------------------------------
184
471
  // HEARTBEAT AND DEGRADED MARKER
@@ -194,10 +481,15 @@ let runs = 0;
194
481
  // backoff between runs, within the healthy band
195
482
  // degraded the crash-loop breaker has tripped: too many exits inside the window, so the fleet
196
483
  // should see a machine that is present but unhealthy rather than one that is gone
484
+ // stopped this generation has given up the lock and exited
485
+ //
486
+ // Two fields are worth reading before the state:
487
+ // orphans_reaped / orphans_reaped_pids what the start-time reap terminated, by pid
488
+ // port.state free | bound | bound-by-other | held | unknown
197
489
  //
198
490
  // Written atomically (temp file + rename) so a reader never sees half a JSON document.
199
491
  // ---------------------------------------------------------------------------------------------
200
- const heartbeatPath = join(dirname(logPath), 'supervisor-heartbeat.json');
492
+ const heartbeatPath = join(runDir, 'supervisor-heartbeat.json');
201
493
  const degradedAfter = Number.isFinite(Number(pick('degraded-after', 3))) ? Number(pick('degraded-after', 3)) : 3;
202
494
  const degradedWindowMs = 10 * 60_000;
203
495
  const degradedBackoffMs = Number.isFinite(Number(pick('degraded-backoff-ms', 300_000)))
@@ -207,18 +499,45 @@ const exitTimes = [];
207
499
 
208
500
  function writeHeartbeat(patch) {
209
501
  const recent = exitTimes.filter((at) => Date.now() - at < degradedWindowMs).length;
210
- const state = patch.state ?? (recent >= degradedAfter ? 'degraded' : 'backoff');
502
+ // With a live child the state is `running`, whoever is asking. The 30 s beat passes no state of its
503
+ // own, and defaulting it to `backoff` published "backoff" alongside a `child_pid` that named a live
504
+ // agent -- measured on this machine at 17:30, with pid 9860 serving `stream ready`. The one file an
505
+ // outside observer reads must not be the one file that contradicts itself.
506
+ const state = patch.state ?? (currentChildPid !== null ? 'running' : recent >= degradedAfter ? 'degraded' : 'backoff');
211
507
  const record = {
212
508
  updated_at: stamp(),
213
509
  state,
214
510
  supervisor_pid: process.pid,
215
511
  run: patch.run ?? runs,
216
- child_pid: patch.child_pid ?? null,
217
- child_started_at: patch.child_started_at ?? null,
512
+ // `currentChildPid`, never `null`, unless the patch says otherwise: this field used to be
513
+ // written as `patch.child_pid ?? null` on the 30 s beat, so a healthy generation with a live
514
+ // agent published `child_pid: null` -- measured on this machine at 17:30 while pid 9860 was
515
+ // serving `stream ready`. A field that is null exactly when the agent is fine is worse than no
516
+ // field: it is what made the old single-pid reap look like it had nothing to do.
517
+ child_pid: patch.child_pid !== undefined ? patch.child_pid : currentChildPid,
518
+ child_started_at: patch.child_started_at !== undefined ? patch.child_started_at : currentChildStartedAt,
218
519
  restarts_total: Math.max(0, runs - 1),
219
520
  restarts_within_10min: recent,
220
521
  last_exit_code: patch.last_exit_code ?? null,
221
522
  last_exit_at: patch.last_exit_at ?? null,
523
+ // Why the last attempt did not end in a running agent, when it did not: one of
524
+ // P2P_PORT_HELD_BY_OTHER, P2P_PORT_NOT_BOUND, or null.
525
+ last_reason: patch.last_reason !== undefined ? patch.last_reason : lastReason,
526
+ // Item 2: what this generation reaped, by pid. Counted across generations in the log.
527
+ orphans_reaped: orphansReaped,
528
+ orphans_reaped_pids: orphansReapedPids,
529
+ // Item 1: who holds the single-instance lock, and how often a dead generation had to be
530
+ // cleaned out of the way.
531
+ lock: { path: lockInfo.path, pid: lockInfo.pid, takeovers_total: lockInfo.takeovers_total, takeovers_this_run: lockInfo.takeovers_this_run },
532
+ // Item 3: the punch port, which is the difference between a machine that is *on* the fleet and
533
+ // one that is merely *listed* on it.
534
+ port: {
535
+ configured: portInfo.configured,
536
+ mode: p2pMode,
537
+ state: portInfo.state,
538
+ owner_pid: portInfo.owner_pid,
539
+ note: portInfo.note,
540
+ },
222
541
  // The known killer, recorded in decimal because that is how Node reports it: 3221225786 is
223
542
  // 0xC000013A, STATUS_CONTROL_C_EXIT. Signed, it is -1073741510.
224
543
  note: 'exit code 3221225786 means the console went away; it is not a crash of the agent logic',
@@ -235,52 +554,512 @@ function writeHeartbeat(patch) {
235
554
  return record;
236
555
  }
237
556
 
238
- writeHeartbeat({ state: 'backoff' });
239
557
  // ---------------------------------------------------------------------------------------------
240
- // START-TIME REAPING -- the cost of detaching the agent, paid here
558
+ // PROCESS IDENTITY: HOW A PROCESS IS RECOGNISED, AND WHY NOT BY A REMEMBERED PID
559
+ //
560
+ // The old reap read `child_pid` out of the heartbeat and killed that one pid. Measured cost: with
561
+ // nine agents alive, eight were invisible to it, so it terminated one and left the rest holding the
562
+ // identity and the punch port. A remembered pid is a fact about the past; what is needed is a fact
563
+ // about the present, so this enumerates the process table and matches on **identity**:
564
+ //
565
+ // * the process is node (`node.exe` or a `node` basename of the actual executable), AND
566
+ // * its command line contains `w2m-localside`, AND
567
+ // * its command line contains this installation's identity string -- the agent's `--state`
568
+ // directory by default, `--reap-match` when a test harness or a second checkout needs its own.
569
+ //
570
+ // Everything a reap touches therefore belongs to *this* agent's identity; an unrelated node process
571
+ // -- a test runner, an editor's language server, DSH itself -- cannot match, because it has neither
572
+ // the agent script name nor this state directory on its command line. The command line, rather than
573
+ // a pid file or a parent link, is the only one of those that survives both deaths this project has
574
+ // measured: `Stop-Process -Force` and a console teardown. A detached child is re-parented (or
575
+ // orphaned) by both, so the parent link is worthless exactly when it is needed, while `--state` was
576
+ // on the command line from the first generation to the last and is what one identity *means*.
577
+ //
578
+ // The second marker (`w2m-agent-supervisor` + this `--config`) identifies superseded supervisor
579
+ // generations. They are reported, and are not killed: a live supervisor's lock is authoritative (the
580
+ // lock already makes a second generation exit), while killing a *live* supervisor by command-line
581
+ // match would race its own spawn. Orphaned agents, which cannot hold a lock, are the leak that
582
+ // actually took the port, and they are killed.
583
+ // ---------------------------------------------------------------------------------------------
584
+
585
+ /** The process table, or null when it cannot be read (then the caller says so rather than guessing). */
586
+ function listProcesses() {
587
+ if (process.platform !== 'win32') {
588
+ // POSIX: `ps` is the table. Kept here so the same reap works on the hosts this file claims to
589
+ // support, but the argument shape is the only thing shared -- the matching above is identical.
590
+ const result = spawnSync('ps', ['-eo', 'pid=,args='], { encoding: 'utf8', windowsHide: true, timeout: 10_000 });
591
+ if (result.status !== 0 || typeof result.stdout !== 'string') return null;
592
+ return result.stdout
593
+ .split(/\r?\n/)
594
+ .map((line) => line.trim())
595
+ .filter(Boolean)
596
+ .map((line) => {
597
+ const at = line.indexOf(' ');
598
+ return { pid: Number(line.slice(0, at)), cmd: line.slice(at + 1) };
599
+ })
600
+ .filter((row) => Number.isInteger(row.pid));
601
+ }
602
+ // ONE query for the whole table, filtered in JavaScript. It used to ask PowerShell for the match
603
+ // and paid for it twice: every filter was another `Get-CimInstance` (each with a CIM round trip)
604
+ // and, worse, the `if`/`elseif` block was joined with `;`, which PowerShell reads as an empty
605
+ // statement where a block is required -- a parse error that `-ErrorAction SilentlyContinue` hid
606
+ // while returning nothing. Nothing is exactly what "no orphans to reap" looks like, so the reap
607
+ // silently did nothing. The identity strings travel in the environment (no quoting rules apply)
608
+ // and every match is made here, where a wrong answer is a visible test failure rather than a
609
+ // missing row.
610
+ const script = [
611
+ "$ErrorActionPreference='SilentlyContinue'",
612
+ "Get-CimInstance Win32_Process -Filter \"Name='node.exe'\" | Select-Object ProcessId,CommandLine | ConvertTo-Csv -NoTypeInformation",
613
+ "Write-Output '#end'",
614
+ ].join('\n');
615
+ const result = spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], {
616
+ encoding: 'utf8',
617
+ windowsHide: true,
618
+ timeout: 20_000,
619
+ });
620
+ if (result.error || typeof result.stdout !== 'string' || !result.stdout.includes('#end')) return null;
621
+ const rows = [];
622
+ for (const line of result.stdout.split(/\r?\n/)) {
623
+ const trimmed = line.trim();
624
+ if (!trimmed.startsWith('"')) continue;
625
+ // CSV from PowerShell: '"pid","command line"', with embedded quotes doubled. The command line
626
+ // itself contains commas, so the pid is taken from the front and the remainder is re-joined
627
+ // rather than split into fields.
628
+ const parts = trimmed.split('","');
629
+ const pid = Number(parts[0].replace(/^"|"$/g, ''));
630
+ if (!Number.isInteger(pid) || pid <= 0) continue;
631
+ rows.push({ pid, cmd: parts.join('","').replace(/^"|"$/g, '').replace(/""/g, '"') });
632
+ }
633
+ trace('process-table', { rows: rows.length });
634
+ return rows;
635
+ }
636
+
637
+ /** Every live process that is this installation's agent, whoever started it. */
638
+ function agentProcesses() {
639
+ const rows = listProcesses();
640
+ if (rows === null) return null;
641
+ const wanted = String(reapMatch).toLowerCase().replace(/\//g, '\\');
642
+ const matched = [];
643
+ for (const row of rows) {
644
+ if (typeof row.cmd !== 'string') continue;
645
+ // A supervisor is never an agent, and this one is never a candidate for anything. Both checks
646
+ // are load-bearing: the identity string is an argument of the supervisor as well as of the agent
647
+ // (`--state <dir>` was in this file's own command line), so matching on identity alone made the
648
+ // supervisor reap *itself* -- measured, and it died between `reap-before-kill` and the kill
649
+ // returning, which is the worst possible version of this bug: a process that kills the agent of
650
+ // a different installation, or itself, is worse than one that leaves orphans behind.
651
+ if (row.pid === process.pid) continue;
652
+ const identity = row.cmd.toLowerCase().replace(/\//g, '\\');
653
+ if (!identity.includes(wanted)) continue;
654
+ if (isSupervisorCommandLine(row.cmd)) {
655
+ trace('reap-skipped-supervisor', { pid: row.pid });
656
+ continue;
657
+ }
658
+ if (!row.cmd.toLowerCase().includes(PROC_MARKER_AGENT)) {
659
+ trace('reap-skipped-no-agent-marker', { pid: row.pid, cmd: row.cmd.slice(0, 160) });
660
+ continue;
661
+ }
662
+ matched.push(row);
663
+ }
664
+ trace('agent-processes', { rows: rows.length, wanted, matched: matched.map((row) => row.pid) });
665
+ return matched;
666
+ }
667
+
668
+ /** Superseded supervisor generations, for the report only -- see the block comment above. */
669
+ function supervisorProcesses() {
670
+ const rows = listProcesses();
671
+ if (rows === null) return null;
672
+ const self = process.pid;
673
+ return rows.filter((row) => row.pid !== self && isSupervisorCommandLine(row.cmd));
674
+ }
675
+
676
+ function isAlive(pid) {
677
+ try {
678
+ process.kill(pid, 0);
679
+ return true;
680
+ } catch {
681
+ return false;
682
+ }
683
+ }
684
+
685
+ /**
686
+ * Terminate one pid, hard on Windows -- and then *check*, because this is the operation that decides
687
+ * whether an orphan is really gone.
688
+ *
689
+ * `process.kill(pid, 'SIGKILL')` cannot end a process this one has no console relationship with, and
690
+ * a Task Scheduler agent has none, so `taskkill /T /F` is the Windows way to end a tree; `/T` matters
691
+ * for a grandchild an agent may have spawned. But `/T` alone is not enough, and its failure is
692
+ * silent: measured, `taskkill /PID <pid of an exited parent> /T /F` exits 128 with "process not
693
+ * found" and kills nothing at all -- and a detached agent is exactly a pid whose parent has already
694
+ * exited. A reap that trusted taskkill's exit code would report success while the orphan kept its
695
+ * claim on the punch port, which is this defect all over again. So: retry, escalate to a direct kill
696
+ * of the pid itself, and return the truth of `isAlive` rather than taskkill's opinion.
697
+ */
698
+ function terminate(pid, timeoutMs = 2_000) {
699
+ // Never this process. `taskkill /PID <self>` succeeds, so a matching bug upstream would end the
700
+ // supervisor silently -- the exact class of death this file exists to make impossible.
701
+ if (pid === process.pid) {
702
+ say(`[w2m-supervisor] refusing to terminate pid=${pid}: that is this supervisor`);
703
+ return false;
704
+ }
705
+ if (process.platform !== 'win32') {
706
+ try {
707
+ process.kill(pid, 'SIGKILL');
708
+ } catch {
709
+ /* already gone */
710
+ }
711
+ return !isAlive(pid);
712
+ }
713
+ for (let attempt = 0; attempt < 5; attempt += 1) {
714
+ if (!isAlive(pid)) return true;
715
+ for (const flags of [['/T', '/F'], ['/F']]) {
716
+ spawnSync('taskkill', ['/PID', String(pid), ...flags], { encoding: 'utf8', windowsHide: true, timeout: timeoutMs });
717
+ }
718
+ }
719
+ return !isAlive(pid);
720
+ }
721
+
722
+ // ---------------------------------------------------------------------------------------------
723
+ // ITEM 2: REAP BY IDENTITY, AT STARTUP
241
724
  //
242
725
  // Detaching the agent is what stops a console teardown from taking the machine off the fleet, and it
243
726
  // has a price that was measured: every supervisor generation that dies without stopping its child
244
- // leaves an orphan behind. Three agents with one identity were found running at once, the newest
245
- // unable to bind the punch port (`P2P_BIND_FAILED: EADDRINUSE 0.0.0.0:41235`) because an orphan held
246
- // it -- two streams on one device token, and a port that pins nothing.
247
- //
248
- // So the supervisor reaps what it did not start: the heartbeat names the last child, and if that pid
249
- // is still alive when a new supervisor starts, it is a previous generation's orphan and is terminated
250
- // before anything else runs. "Trust a graceful exit" is exactly the assumption that failed.
727
+ // leaves an orphan behind, and a five-minute watchdog makes a new generation per tick. The reap
728
+ // below is the payment: it looks at every agent process of this identity and keeps none of them,
729
+ // because at this point in the file this generation has the lock and has not spawned anything, so
730
+ // **any** agent process alive right now belongs to a dead generation. "Trust a graceful exit" is the
731
+ // assumption that failed; "trust the process table" is the one that holds.
251
732
  // ---------------------------------------------------------------------------------------------
733
+ let startupReapNote = 'not attempted';
252
734
  try {
253
- const previous = JSON.parse(readFileSync(heartbeatPath, 'utf8').replace(/^\uFEFF/, ''));
254
- const orphan = Number(previous.child_pid);
255
- if (Number.isInteger(orphan) && orphan > 0 && orphan !== process.pid) {
256
- let alive = true;
735
+ const stale = agentProcesses();
736
+ if (stale === null) {
737
+ startupReapNote = 'could not enumerate the process table; nothing reaped';
738
+ say(`[w2m-supervisor] ${startupReapNote}`);
739
+ } else {
740
+ const killedPids = [];
741
+ const survivors = [];
742
+ for (const row of stale) {
743
+ if (row.pid === currentChildPid) continue; // impossible before the first spawn, but never kill our own child
744
+ const killed = terminate(row.pid);
745
+ if (killed) killedPids.push(row.pid);
746
+ else survivors.push(row.pid);
747
+ say(
748
+ `[w2m-supervisor] reaped orphan agent pid=${row.pid}${killed ? '' : " (would not die; it may belong to another user)"}` +
749
+ ` cmd=${row.cmd.slice(0, 200)}`,
750
+ );
751
+ }
752
+ orphansReaped = killedPids.length;
753
+ orphansReapedPids = killedPids;
754
+ startupReapNote =
755
+ stale.length === 0
756
+ ? 'no orphan agents found'
757
+ : `reaped ${killedPids.length} of ${stale.length}${survivors.length > 0 ? ` (survived: pids=[${survivors.join(',')}])` : ''}`;
758
+ say(
759
+ `[w2m-supervisor] start-time reap by identity (state=${state}): ${startupReapNote}` +
760
+ `${killedPids.length > 0 ? ` pids=[${killedPids.join(',')}]` : ''}`,
761
+ );
762
+ }
763
+ } catch (error) {
764
+ startupReapNote = `reap failed: ${error.message}`;
765
+ say(`[w2m-supervisor] ${startupReapNote}`);
766
+ }
767
+ // The heartbeat is written *before* the first spawn so the reap and the port decision are visible
768
+ // even if this generation dies in the next second -- a fact in a file outlives a process.
769
+ const startupSuperseded = (() => {
770
+ try {
771
+ const others = supervisorProcesses();
772
+ return others === null ? null : others.length;
773
+ } catch {
774
+ return null;
775
+ }
776
+ })();
777
+ if (startupSuperseded !== null && startupSuperseded > 0) {
778
+ say(
779
+ `[w2m-supervisor] note: ${startupSuperseded} other supervisor process(es) of this identity are alive; ` +
780
+ `the lock -- not a kill -- is what stops them from supervising anything. Reported, not killed: ` +
781
+ `ending a live supervisor here would race the agent it may be about to own.`,
782
+ );
783
+ }
784
+ trace('startup-reap', { orphans_reaped: orphansReaped, pids: orphansReapedPids, note: startupReapNote });
785
+ writeHeartbeat({ state: 'backoff' });
786
+
787
+ // ---------------------------------------------------------------------------------------------
788
+ // ITEM 3: A HELD PUNCH PORT IS A FATAL STARTUP ERROR, NOT A QUIET DEGRADATION
789
+ //
790
+ // The agent's own design is deliberate and correct as library behaviour: `startP2P()` records
791
+ // `P2P_BIND_FAILED` and returns `{ok:false}` because a machine whose fast path is broken is still a
792
+ // working machine over the relay, and the agent's comment says so: "never fatal". The fleet measured
793
+ // what that costs when the port is held by an *orphan of the same identity*: nine agents ran, eight
794
+ // of them with no direct path, every dispatch fell back to the relay with `P2P_UNREACHABLE`, and
795
+ // because a degraded agent is indistinguishable from a healthy one from outside -- same log line
796
+ // volume, same stream, `state: running` -- nothing on this machine or on the fleet noticed for
797
+ // fourteen hours.
798
+ //
799
+ // So the choice between (a) fatal and (b) degraded-but-visible is made here as (a), for these
800
+ // reasons:
801
+ // * The supervisor can decide it *before* spawning, which is the only place the information is
802
+ // clean: `probePort()` binds the punch port itself. If the bind succeeds, the port is free and
803
+ // the agent's own bind cannot fail for the one reason that has ever been observed here; if it
804
+ // fails, something holds the port and the agent's bind will fail in exactly the same way.
805
+ // * (b) cannot be implemented honestly from out here. The supervisor cannot see the agent's
806
+ // in-process `p2pStartResult`, and parsing the agent's log for `P2P_BIND_FAILED` makes a log
807
+ // line load-bearing -- the file this whole design exists because a log line is not guaranteed
808
+ // to be written. A heartbeat that says "degraded" only when a log happened to be flushed is a
809
+ // heartbeat that lies during the incident it is for.
810
+ // * Retrying is what an operator would do, and it is safe to automate *because of the lock*: with
811
+ // one generation possible, a retry cannot produce a second agent, so "reap and retry" costs one
812
+ // backoff and removes the accumulated state rather than compounding it.
813
+ // The state is not hidden: the heartbeat carries `port.state` (free/bound/bound-by-other/held) and a
814
+ // named `last_reason` (P2P_PORT_HELD_BY_OTHER / P2P_PORT_NOT_BOUND), so "the agent is up but has no
815
+ // punch port" can never again be a silent condition -- it is either repaired or said out loud.
816
+ // ---------------------------------------------------------------------------------------------
817
+
818
+ /**
819
+ * Can this process bind the punch port? The probe is the agent's own setup (`udp4`, `0.0.0.0`), so
820
+ * its answer is the agent's answer. A successful bind is closed immediately: it is a question, not
821
+ * a claim on the port.
822
+ *
823
+ * @returns {Promise<{free: boolean, code: string|null}>}
824
+ */
825
+ function probePort(port) {
826
+ return new Promise((resolve) => {
827
+ let settled = false;
828
+ const done = (result) => {
829
+ if (settled) return;
830
+ settled = true;
831
+ resolve(result);
832
+ };
833
+ let socket;
257
834
  try {
258
- process.kill(orphan, 0);
259
- } catch {
260
- alive = false;
835
+ socket = createSocket('udp4');
836
+ } catch (error) {
837
+ done({ free: false, code: error.code ?? error.message });
838
+ return;
261
839
  }
262
- if (alive) {
840
+ const timer = setTimeout(() => {
841
+ try {
842
+ socket.close();
843
+ } catch {
844
+ /* never bound */
845
+ }
846
+ done({ free: false, code: 'ETIMEDOUT' });
847
+ }, 3_000);
848
+ timer.unref?.();
849
+ socket.on('error', (error) => {
850
+ clearTimeout(timer);
851
+ done({ free: false, code: error.code ?? error.message });
852
+ });
853
+ socket.bind({ address: '0.0.0.0', port }, () => {
854
+ clearTimeout(timer);
263
855
  try {
264
- process.kill(orphan);
265
- say(`[w2m-supervisor] reaped orphan agent pid=${orphan} left by an earlier supervisor`);
266
- await new Promise((resolve) => {
267
- setTimeout(resolve, 1500);
268
- });
269
- } catch (error) {
270
- say(`[w2m-supervisor] could not reap orphan pid=${orphan}: ${error.message}`);
856
+ socket.close();
857
+ } catch {
858
+ /* closing a bound socket cannot fail meaningfully */
859
+ }
860
+ done({ free: true, code: null });
861
+ });
862
+ });
863
+ }
864
+
865
+ function notePort(state_, ownerPid, note) {
866
+ portInfo.state = state_;
867
+ portInfo.owner_pid = ownerPid;
868
+ portInfo.note = note;
869
+ }
870
+
871
+ /**
872
+ * The pid holding the port, when the OS will say. Diagnostic only: absence is not a failure.
873
+ *
874
+ * `netstat -ano` rather than `Get-NetUDPEndpoint`: measured on this machine, one PowerShell start is
875
+ * ~2.6 s against ~0.09 s for netstat, and this runs inside the startup path where a slow answer is
876
+ * indistinguishable from a hung agent. The cmdlet is the fallback for a host without netstat, and
877
+ * `withName` asks for the process name too -- worth 2.6 s only when a human has to read the answer,
878
+ * which is exactly when a foreign process holds the port.
879
+ */
880
+ function portOwner(withName = false) {
881
+ if (process.platform !== 'win32') return null;
882
+ const fromNetstat = () => {
883
+ const result = spawnSync('netstat', ['-ano', '-p', 'UDP'], { encoding: 'utf8', windowsHide: true, timeout: 10_000 });
884
+ if (result.error || typeof result.stdout !== 'string') return null;
885
+ for (const line of result.stdout.split(/\r?\n/)) {
886
+ const match = /^\s*UDP\s+\S*:(\d+)\s+\S+\s+(\d+)\s*$/.exec(line);
887
+ if (match && Number(match[1]) === p2pPort) return { pid: Number(match[2]), name: null };
888
+ }
889
+ return null;
890
+ };
891
+ try {
892
+ const quick = fromNetstat();
893
+ if (quick !== null && !withName) return quick;
894
+ // Ask the cmdlet once: it names the process, and it is the only route when netstat is absent or
895
+ // when its output has no line for this port at all.
896
+ const script = `$e=Get-NetUDPEndpoint -LocalPort ${p2pPort} -ErrorAction SilentlyContinue | Select-Object -First 1; if ($e) { $p=Get-Process -Id $e.OwningProcess -ErrorAction SilentlyContinue; "$($e.OwningProcess) $($p.ProcessName)" }`;
897
+ const result = spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], {
898
+ encoding: 'utf8',
899
+ windowsHide: true,
900
+ timeout: 10_000,
901
+ });
902
+ const text = typeof result.stdout === 'string' ? result.stdout.trim() : '';
903
+ const [pidText, name] = text.split(/\s+/);
904
+ const pid = Number(pidText);
905
+ if (Number.isInteger(pid) && pid > 0) return { pid, name: name ?? 'unknown' };
906
+ return quick;
907
+ } catch {
908
+ return null;
909
+ }
910
+ }
911
+
912
+ const killedByUs = (pid) => typeof pid === 'number' && pid === currentChildPid;
913
+
914
+ /**
915
+ * Wait for the punch port to be free, reaping every other agent of this identity while waiting.
916
+ *
917
+ * UDP has no TIME_WAIT, so a killed holder releases the port as soon as its process is gone; the
918
+ * wait is for `taskkill` to finish, not for a timer. The one case this cannot repair is a foreign
919
+ * process on the same port, and that is reported by name rather than swallowed.
920
+ *
921
+ * @returns {Promise<{free: boolean, reason: string|null, heldBy: object|null}>}
922
+ */
923
+ async function ensurePortFree() {
924
+ trace('port-gate', { port: p2pPort, mode: p2pMode });
925
+ if (p2pPort === null || p2pMode === 'relay') {
926
+ notePort('not-configured', null, p2pPort === null ? 'no pinned punch port; the agent picks an ephemeral one' : 'p2p mode is relay');
927
+ return { free: true, reason: null, heldBy: null };
928
+ }
929
+ portInfo.configured = p2pPort;
930
+ let probe = await probePort(p2pPort);
931
+ trace('port-gate-probe', { free: probe.free, code: probe.code });
932
+ if (probe.free) {
933
+ notePort('free', null, 'the punch port was free before the spawn');
934
+ return { free: true, reason: null, heldBy: null };
935
+ }
936
+ // Something holds it. If it is an agent of this identity, it is an orphan that the start-time reap
937
+ // should already have taken; a second pass covers the race where it appeared afterwards.
938
+ const holder = portOwner();
939
+ const agents = agentProcesses();
940
+ const agentPids = agents === null ? [] : agents.map((row) => row.pid);
941
+ const extra = agentPids.filter((pid) => !killedByUs(pid));
942
+ if (extra.length > 0) {
943
+ say(`[w2m-supervisor] punch port ${p2pPort} is held and ${extra.length} agent process(es) of this identity are alive: pids=[${extra.join(',')}] -- reaping and retrying`);
944
+ const killedNow = [];
945
+ for (const pid of extra) {
946
+ if (terminate(pid)) killedNow.push(pid);
947
+ }
948
+ orphansReaped += killedNow.length;
949
+ orphansReapedPids = [...new Set(orphansReapedPids.concat(killedNow))];
950
+ }
951
+ for (let attempt = 0; attempt < 15; attempt += 1) {
952
+ await new Promise((resolve) => {
953
+ setTimeout(resolve, 200);
954
+ });
955
+ probe = await probePort(p2pPort);
956
+ trace('port-gate-retry', { attempt, free: probe.free, code: probe.code });
957
+ if (probe.free) {
958
+ notePort('free', null, `the punch port came free after the reap${extra.length > 0 ? ` of pids=[${extra.join(',')}]` : ''}`);
959
+ return { free: true, reason: null, heldBy: null };
960
+ }
961
+ }
962
+ // Still held after every agent of this identity was taken out: it is somebody else's socket. No
963
+ // amount of retrying will change that, and starting the agent anyway is the fourteen-hour failure.
964
+ const owner = holder ?? portOwner(true);
965
+ notePort('bound-by-other', owner?.pid ?? null, `the punch port is held by ${owner ? `pid=${owner.pid} (${owner.name})` : 'another process'}, which is not an agent of this identity`);
966
+ return {
967
+ free: false,
968
+ reason: 'P2P_PORT_HELD_BY_OTHER',
969
+ heldBy: owner,
970
+ };
971
+ }
972
+
973
+ /** Give up on the port and let the named reason reach the heartbeat before the next backoff. */
974
+ async function refuseToSpawn(reason, heldBy) {
975
+ lastReason = reason;
976
+ const detail =
977
+ reason === 'P2P_PORT_HELD_BY_OTHER'
978
+ ? `punch port ${p2pPort} is held by ${heldBy ? `pid=${heldBy.pid} (${heldBy.name})` : 'an unidentified process'} and it is not an agent of this identity`
979
+ : `the agent did not bind punch port ${p2pPort} within the deadline`;
980
+ say(`[w2m-supervisor] refusing to start: ${reason}: ${detail} -- the agent would run with no direct path and every dispatch would fall back to the relay`);
981
+ if (portInfo.state !== 'bound-by-other') notePort('held', heldBy?.pid ?? null, detail);
982
+ writeHeartbeat({ state: 'degraded', last_reason: reason, child_pid: null, child_started_at: null });
983
+ }
984
+
985
+ /**
986
+ * Did the agent actually take the punch port?
987
+ *
988
+ * `probePort` answers "is the port free": after the spawn, free means the agent has not bound it. As
989
+ * the agent's own bind is not observable from here (it is in-process), this is the honest check the
990
+ * supervisor can make, and it turns the one silent degradation that has actually happened into a
991
+ * named restart: if the port is still free after the deadline, the agent is running without a direct
992
+ * path, so it is stopped and reported rather than left to look healthy.
993
+ *
994
+ * @returns {Promise<{bound: boolean, code: string|null}>}
995
+ */
996
+ async function waitForPortBound(childPid, deadlineMs) {
997
+ const began = Date.now();
998
+ for (;;) {
999
+ const probe = await probePort(p2pPort);
1000
+ trace('port-after-spawn-probe', { pid: childPid, free: probe.free, code: probe.code });
1001
+ if (!probe.free) {
1002
+ const owner = portOwner();
1003
+ trace('port-after-spawn-owner', { owner: owner?.pid ?? null, name: owner?.name ?? null });
1004
+ if (owner === null) {
1005
+ // Bound, but the OS would not name the holder. The bind succeeded for us one moment ago and
1006
+ // the agent is alive, so the honest reading is "the agent took it"; `portOwner` is a
1007
+ // diagnostic, and its absence must not turn a working agent into a restart loop.
1008
+ notePort('bound', childPid, `punch port ${p2pPort} is bound and the agent (pid=${childPid}) is alive; the OS did not name the holder`);
1009
+ return { bound: true, code: null };
271
1010
  }
1011
+ if (owner.pid === childPid) {
1012
+ notePort('bound', owner.pid, `the punch port is bound by the agent (pid=${owner.pid})`);
1013
+ return { bound: true, code: null };
1014
+ }
1015
+ notePort('bound-by-other', owner.pid, `the punch port is bound by pid=${owner.pid} (${owner.name}), not by the agent pid=${childPid}`);
1016
+ return { bound: false, code: 'P2P_PORT_HELD_BY_OTHER' };
272
1017
  }
1018
+ if (!isAlive(childPid) || Date.now() - began >= deadlineMs) return { bound: false, code: probe.code };
1019
+ await new Promise((resolve) => {
1020
+ setTimeout(resolve, 500);
1021
+ });
273
1022
  }
274
- } catch {
275
- // No heartbeat yet, or unreadable: nothing to reap, and that is not an error worth stopping for.
276
1023
  }
1024
+
1025
+ async function stopChild(child, why) {
1026
+ if (!child || child.pid === undefined) return;
1027
+ if (!isAlive(child.pid)) return;
1028
+ say(`[w2m-supervisor] stopping agent pid=${child.pid} (${why})`);
1029
+ terminate(child.pid);
1030
+ }
1031
+
277
1032
  // A periodic beat, so "the supervisor itself is alive but a worker is wedged" is visible too.
278
- const beat = setInterval(() => writeHeartbeat({}), 30_000);
1033
+ const beat = setInterval(() => {
1034
+ trace('beat', { port_state: portInfo.state, child: currentChildPid });
1035
+ writeHeartbeat({});
1036
+ }, 30_000);
279
1037
  beat.unref();
280
1038
 
1039
+ const portWaitMs = Number.isFinite(Number(pick('port-wait-ms', 30_000))) ? Number(pick('port-wait-ms', 30_000)) : 30_000;
1040
+
281
1041
  while (!stopping) {
282
1042
  runs += 1;
283
1043
  const startedAt = Date.now();
1044
+ trace('loop-iteration', { runs, stopping, currentChildPid });
1045
+ // Item 3, first half: never spawn into a port that is already taken. This is where an orphan from
1046
+ // a previous generation is discovered even if it appeared after the start-time reap.
1047
+ const gate = await ensurePortFree();
1048
+ if (stopping) break;
1049
+ if (!gate.free) {
1050
+ await refuseToSpawn(gate.reason, gate.heldBy);
1051
+ exitTimes.push(Date.now());
1052
+ const record = writeHeartbeat({});
1053
+ const wait = record.state === 'degraded' ? degradedBackoffMs : backoffMs;
1054
+ // No hot loop here: the port gate is paced by the same backoff a crashed agent gets, so a
1055
+ // persistently occupied port produces a slow, visible, well-labelled heartbeat rather than a
1056
+ // spawn storm.
1057
+ say(`[w2m-supervisor] run ${runs} refused to start (${gate.reason}) -- waiting ${Math.round(wait / 1000)}s`);
1058
+ await new Promise((resolve) => {
1059
+ setTimeout(resolve, wait);
1060
+ });
1061
+ continue;
1062
+ }
284
1063
  // ---------------------------------------------------------------------------------------------
285
1064
  // THE AGENT MUST NOT SHARE THIS PROCESS'S CONSOLE.
286
1065
  //
@@ -309,15 +1088,63 @@ while (!stopping) {
309
1088
  stdio: ['ignore', logFd, logFd],
310
1089
  windowsHide: true,
311
1090
  });
312
- writeHeartbeat({ state: 'running', child_pid: child.pid ?? null, child_started_at: stamp() });
1091
+ currentChildPid = child.pid ?? null;
1092
+ currentChildStartedAt = stamp();
1093
+ lastReason = null;
1094
+ if (child.pid === undefined) {
1095
+ // No pid at all: there is nothing to own and nothing to check, so this is one named run rather
1096
+ // than a spawn followed by a 30 s port wait that could only ever conclude "not bound".
1097
+ exitTimes.push(Date.now());
1098
+ const failed = await exited;
1099
+ say(`[w2m-supervisor] run ${runs} never started (spawn returned no pid); exit code=${failed.exitCode} -- restarting`);
1100
+ const record = writeHeartbeat({ last_exit_code: failed.exitCode, last_exit_at: stamp(), last_reason: 'SPAWN_FAILED', child_pid: null, child_started_at: null });
1101
+ await new Promise((resolve) => {
1102
+ setTimeout(resolve, record.state === 'degraded' ? degradedBackoffMs : backoffMs);
1103
+ });
1104
+ continue;
1105
+ }
1106
+ writeHeartbeat({ state: 'running', child_pid: currentChildPid, child_started_at: currentChildStartedAt });
1107
+ say(`[w2m-supervisor] run ${runs} started agent pid=${currentChildPid}`);
1108
+ trace('spawned', { pid: currentChildPid });
313
1109
 
314
- const result = await new Promise((resolve) => {
1110
+ // Item 3, second half: confirm the agent really took the port, and stop it if it did not. Two
1111
+ // checks run concurrently, because either is a reason to stop: the port deadline, and the child's
1112
+ // own exit (the loop below must not hold a dead child's heartbeat open as `running`).
1113
+ const exited = new Promise((resolve) => {
315
1114
  child.on('exit', (exitCode, signal) => resolve({ exitCode, signal }));
316
1115
  child.on('error', (error) => {
317
1116
  say(`[w2m-supervisor] spawn failed: ${error.message}`);
318
1117
  resolve({ exitCode: null, signal: null });
319
1118
  });
320
1119
  });
1120
+ const portCheck =
1121
+ p2pPort === null || p2pMode === 'relay'
1122
+ ? Promise.resolve({ bound: true, code: null })
1123
+ : waitForPortBound(child.pid, portWaitMs);
1124
+ const winner = await Promise.race([
1125
+ portCheck.then((result) => ({ kind: 'port', result })),
1126
+ exited.then((result) => ({ kind: 'exit', result })),
1127
+ ]).catch((error) => {
1128
+ // A rejection here would otherwise be the one death this file cannot describe: Node ends the
1129
+ // process on an unhandled rejection, so the supervisor would disappear with nothing but a stack
1130
+ // trace. Named, logged, and turned into a decision instead.
1131
+ say(`[w2m-supervisor] the punch-port check failed: ${error.stack ?? error.message}`);
1132
+ trace('port-check-threw', { message: error.message });
1133
+ return { kind: 'exit', result: { exitCode: null, signal: null } };
1134
+ });
1135
+ if (winner.kind === 'port' && !winner.result.bound && isAlive(child.pid)) {
1136
+ // The port went to somebody else between the gate and the bind: the agent is alive but had no
1137
+ // direct path, which is the exact condition that used to run silently for fourteen hours.
1138
+ const reason = winner.result.code === 'P2P_PORT_HELD_BY_OTHER' ? 'P2P_PORT_HELD_BY_OTHER' : 'P2P_PORT_NOT_BOUND';
1139
+ if (reason === 'P2P_PORT_NOT_BOUND') {
1140
+ notePort('held', portOwner()?.pid ?? null, `nothing bound punch port ${p2pPort} within ${Math.round(portWaitMs / 1000)}s of the spawn`);
1141
+ }
1142
+ await refuseToSpawn(reason, portInfo.owner_pid === null ? null : { pid: portInfo.owner_pid, name: 'unknown' });
1143
+ await stopChild(child, `${reason}: it had no punch port, so every dispatch would use the relay`);
1144
+ }
1145
+ const result = await exited;
1146
+ if (currentChildPid === child.pid) currentChildPid = null;
1147
+ currentChildStartedAt = null;
321
1148
  exitTimes.push(Date.now());
322
1149
 
323
1150
  const lived = Math.round((Date.now() - startedAt) / 1000);
@@ -344,4 +1171,4 @@ while (!stopping) {
344
1171
  });
345
1172
  }
346
1173
  writeHeartbeat({ state: 'stopped' });
347
- log.end();
1174
+ dropLock('this generation is exiting');