@bridge4dev/runner 0.45.1 → 0.47.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,8 +14,14 @@ import { VerifyRunner, runOneOffCommand, } from './verify.js';
14
14
  import { VerifyReportQueue } from './verify-queue.js';
15
15
  import { applySession, gitBranches, gitCommit, gitDiff, gitLog, gitPush, gitRefs, gitShow, gitStatus, revertApply, gitStage, gitUnstage, gitDiscard, gitPull, gitMergeAbort, updateFromBase, workspaceState, } from './gitops.js';
16
16
  import { fsView } from './fsview.js';
17
+ import { publishFile } from './file-publish.js';
17
18
  import { agentAuthStatuses, AuthRelay, clearAgentAuthFailure, noteAgentAuthFailure, } from './auth-relay.js';
18
19
  import { selfUpdate } from './self-update.js';
20
+ import { installAgent } from './agent-install.js';
21
+ import { autoUpdateRefusal, claimAgentAutoUpdate } from './agent-auto-update.js';
22
+ import { pruneNativeClaudeVersions } from './agent-cleanup.js';
23
+ import { agentByDbValue } from './agent-registry.js';
24
+ import { invalidateAgentVersions, measureAgentVersions, } from './agent-versions.js';
19
25
  import { rememberWorkspacePath } from './environment.js';
20
26
  import { composeMessageWithAttachments, saveAttachments, } from './attachments.js';
21
27
  import { applyRewind, createCheckpoint, dropCheckpoints, listCheckpoints, previewRewind, pruneCheckpoints, } from './checkpoints.js';
@@ -107,8 +113,20 @@ export class Supervisor {
107
113
  * than having no restore point for that one message.
108
114
  */
109
115
  repoLockDepth = new Map();
110
- /** An update is installing right now — a second one would fight it. */
111
- selfUpdateInFlight = false;
116
+ /**
117
+ * One install at a time on this machine, whoever asked for it.
118
+ *
119
+ * `self_update` and `agent_install` both end in `npm install -g` into the
120
+ * SAME npm prefix under the dedicated-user layout, so two of them at once is
121
+ * a real conflict rather than a theoretical one. The API takes a Redis lock
122
+ * before pressing either button, but that lock cannot cover an install the
123
+ * runner starts by itself (stage D) — and the runner has no Redis. This is
124
+ * the only lock that sees both.
125
+ *
126
+ * Holds the holder's name rather than a boolean, so the refusal can say
127
+ * which of the two is running.
128
+ */
129
+ installInFlight = null;
112
130
  /** Session 14: one project-recipe run per machine, and its verdict queue. */
113
131
  verify;
114
132
  verifyReports = new VerifyReportQueue();
@@ -131,7 +149,313 @@ export class Supervisor {
131
149
  ws.on('frame', (frame) => {
132
150
  void this.onFrame(frame).catch((error) => log.error('supervisor: frame handler failed', { type: frame.type, error: String(error) }));
133
151
  });
152
+ /**
153
+ * The seat report is on a heartbeat, not only on the events that change it.
154
+ *
155
+ * Deliberate, and the lesson of the bug this whole frame exists for: seat
156
+ * state lives in three mutable fields on each entry (`session`,
157
+ * `stopRequested`, `parkRequested`), so there is no single mutation to hook
158
+ * — and «every call site remembers to tell the API» is exactly the rule
159
+ * that had already been broken in three places. A tick that re-derives the
160
+ * truth cannot be forgotten by a future call site.
161
+ *
162
+ * Nearly free: `publishSlots` fingerprints what it last sent and returns
163
+ * without touching the socket when nothing moved, so a quiet runner sends
164
+ * one frame per change and none in between.
165
+ */
166
+ this.slotsTimer = setInterval(() => this.publishSlots(), Supervisor.SLOTS_REPORT_INTERVAL_MS);
167
+ this.slotsTimer.unref?.();
168
+ /**
169
+ * Agent versions on a heartbeat too, for a different reason than seats.
170
+ *
171
+ * Nothing on this machine tells us when somebody updates Codex by hand, and
172
+ * the measurement behind the frame is cached for an hour anyway — so the
173
+ * tick costs a cached read most of the time and still notices a hand-made
174
+ * change within the hour. What makes a freshly connected card correct
175
+ * immediately is the same frame sent on every `hello_ack`.
176
+ */
177
+ this.agentVersionsTimer = setInterval(() => void this.publishAgentVersions('measured'), Supervisor.AGENT_VERSIONS_INTERVAL_MS);
178
+ this.agentVersionsTimer.unref?.();
179
+ /**
180
+ * Р12: the old Claude version files, swept once a day.
181
+ *
182
+ * Two timers rather than one, and the delayed first sweep is the load-bearing
183
+ * half. The gate below is «this runner is tracking no sessions», and at the
184
+ * moment a daemon starts that is trivially true — the map fills from the
185
+ * `hello_ack` reconciliation seconds later. Sweeping in that window would be
186
+ * sweeping exactly when the answer is least trustworthy, on the one machine
187
+ * shape (restarted often) where it would happen every time.
188
+ */
189
+ const cleanupFirst = opts.agentCleanupMs ?? Supervisor.AGENT_CLEANUP_FIRST_DELAY_MS;
190
+ const cleanupEvery = opts.agentCleanupMs ?? Supervisor.AGENT_CLEANUP_INTERVAL_MS;
191
+ this.agentCleanupFirstTimer = setTimeout(() => this.sweepOldAgentVersions(), cleanupFirst);
192
+ this.agentCleanupFirstTimer.unref?.();
193
+ this.agentCleanupTimer = setInterval(() => this.sweepOldAgentVersions(), cleanupEvery);
194
+ this.agentCleanupTimer.unref?.();
195
+ }
196
+ /** How often the agent versions are re-derived. See the constructor. */
197
+ static AGENT_VERSIONS_INTERVAL_MS = 60 * 60 * 1_000;
198
+ agentVersionsTimer;
199
+ /**
200
+ * Tell the API which agents this machine has and at which versions.
201
+ *
202
+ * Never throws and never blocks its caller: a machine where a probe hangs
203
+ * must still start sessions. A frame that could not be written is simply not
204
+ * sent — the next `hello_ack` or the hourly tick carries the same fact, and
205
+ * the measurement behind it is cached, so retrying is nearly free.
206
+ */
207
+ /**
208
+ * Why an install cannot start right now, in words for the person who pressed.
209
+ *
210
+ * `null` means the way is clear.
211
+ */
212
+ installBusyReason() {
213
+ if (this.installInFlight === 'self_update')
214
+ return 'An update is already running';
215
+ if (this.installInFlight === 'agent_install') {
216
+ return 'An agent is already being installed on this server';
217
+ }
218
+ return null;
219
+ }
220
+ /**
221
+ * A version change that has happened but has not reached the API yet.
222
+ *
223
+ * `ws.send` returns false when the socket is down, and an install finishing
224
+ * during a reconnect is not a rare case — it is a 300 MB download that takes
225
+ * minutes. The VERSION recovers on its own (the hourly tick and every
226
+ * `hello_ack` re-send the measurement), but `reason` and `changed` are the
227
+ * audit line itself: dropped, «why is my Codex suddenly different» has no
228
+ * answer anywhere, which is precisely what Р14 exists to prevent.
229
+ *
230
+ * Kept in memory rather than on disk on purpose. A daemon that died mid-install
231
+ * has a bigger hole than one lost line, and persisting it would risk filing an
232
+ * install that a rollback then undid.
233
+ */
234
+ pendingVersionChanges = [];
235
+ /**
236
+ * Two installs cannot overlap — the local lock sees to that — but two can
237
+ * both FINISH inside one outage, and each is a line the log owes. A queue
238
+ * rather than a slot, because the second would otherwise overwrite the first
239
+ * and the older install would be the one that vanished.
240
+ *
241
+ * Bounded: an outage long enough to strand nine installs is one where the
242
+ * missing audit lines are not the problem worth solving.
243
+ */
244
+ static MAX_PENDING_VERSION_CHANGES = 8;
245
+ async publishAgentVersions(reason, changed) {
246
+ try {
247
+ const measure = this.opts.measureAgentVersions ?? measureAgentVersions;
248
+ // After an install — ours or the auto one — the cached number is a lie,
249
+ // and a stale `at` with it would be dropped by the API's ordering guard
250
+ // together with the audit line the install owes. Forcing it here means a
251
+ // caller cannot forget to invalidate the cache first.
252
+ const measurement = await measure({ force: reason !== 'measured' });
253
+ // A plain tick carries an earlier install's news rather than replacing it.
254
+ // The API keys the audit line by the measurement's `at`, so the same
255
+ // change arriving under a later `at` is one line, not two.
256
+ //
257
+ // `changed` is only ever handed over by an install, and an install always
258
+ // calls this with `manual` or `auto` — never `measured`.
259
+ if (changed && reason !== 'measured') {
260
+ this.queueVersionChange({ reason, changed });
261
+ }
262
+ const carried = this.pendingVersionChanges[0];
263
+ const sent = this.ws.send({
264
+ type: 'agent_versions',
265
+ at: measurement.at,
266
+ reason: carried?.reason ?? reason,
267
+ agents: measurement.agents,
268
+ ...(carried ? { changed: carried.changed } : {}),
269
+ });
270
+ // Dropped only once a frame is genuinely written.
271
+ if (sent && carried) {
272
+ this.pendingVersionChanges.shift();
273
+ // One line per frame, so a backlog needs one frame each. Recursing only
274
+ // after a SUCCESSFUL send is what makes this terminate: the queue is
275
+ // shorter every time, and a failed write leaves the loop instead.
276
+ if (this.pendingVersionChanges.length > 0) {
277
+ void this.publishAgentVersions('measured');
278
+ }
279
+ }
280
+ }
281
+ catch (error) {
282
+ if (changed && reason !== 'measured')
283
+ this.queueVersionChange({ reason, changed });
284
+ log.warn('supervisor: could not report agent versions', { error: String(error) });
285
+ }
286
+ }
287
+ queueVersionChange(entry) {
288
+ if (this.pendingVersionChanges.length >= Supervisor.MAX_PENDING_VERSION_CHANGES) {
289
+ log.warn('supervisor: dropping the oldest unreported version change', {
290
+ agent: this.pendingVersionChanges[0]?.changed.agent,
291
+ });
292
+ this.pendingVersionChanges.shift();
293
+ }
294
+ this.pendingVersionChanges.push(entry);
295
+ }
296
+ /**
297
+ * Р13: the agent of a starting session, moved forward in the background.
298
+ *
299
+ * The whole mechanism hangs off ONE fact — whether the API put a version in
300
+ * the descriptor. It does that only for machines whose «Auto-update agents»
301
+ * switch is on, so there is no second copy of the setting here to disagree
302
+ * with it, and a runner that is told nothing does nothing.
303
+ *
304
+ * Never throws and never reports anything into the session feed. A person who
305
+ * pressed «start» asked for a session; a failed background download is not
306
+ * their business, and a scary red line about one would be worse than the
307
+ * silence. Where it IS visible is the server card: the `agent_versions` frame
308
+ * at the end goes out after every outcome, so a machine that ended up without
309
+ * a working agent says so within seconds rather than at the next hourly tick.
310
+ */
311
+ async maybeAutoUpdateAgent(descriptor) {
312
+ const latest = descriptor.agentLatestVersion;
313
+ // The switch. Absent = off, and that covers «the card says no», «we never
314
+ // managed to ask a registry» and «the API predates this release» alike.
315
+ if (!latest)
316
+ return;
317
+ // The machine owner's veto over installing agents at all covers this too:
318
+ // it is the same download onto the same disk, asked for by us instead of by
319
+ // a person. Refusing the button and allowing the timer would be a hole in a
320
+ // control whose entire point is that the server cannot talk past it.
321
+ if (this.opts.agentInstallEnabled === false)
322
+ return;
323
+ const runtime = agentByDbValue(descriptor.agent);
324
+ if (!runtime)
325
+ return;
326
+ try {
327
+ // The CACHED measurement, not a fresh probe.
328
+ //
329
+ // This runs on every session start, and a fresh probe is two child
330
+ // processes per start — `claude --version` plus `claude doctor` — on the
331
+ // machine somebody is trying to start work on. The cache is an hour old at
332
+ // worst and the throttle below is a day, so staleness cannot cost a missed
333
+ // update; and being wrong the other way costs nothing either, because
334
+ // `installAgent` measures again for real and answers «already at that
335
+ // version» without touching anything.
336
+ const measure = this.opts.measureAgentVersions ?? measureAgentVersions;
337
+ const snapshot = await measure();
338
+ const measured = snapshot.agents[runtime.wireKey] ?? {
339
+ version: null,
340
+ managedBy: 'unknown',
341
+ };
342
+ const refusal = autoUpdateRefusal({ runtime, measured, latest });
343
+ if (refusal) {
344
+ log.debug('agent-auto-update: leaving it alone', {
345
+ agent: runtime.wireKey,
346
+ reason: refusal,
347
+ });
348
+ return;
349
+ }
350
+ // The lock is read BEFORE the day is claimed, and the order matters: three
351
+ // sessions starting at once are the ordinary case here, and spending the
352
+ // claim on an attempt that the lock then refuses would cost the agent its
353
+ // whole day for an install that never began.
354
+ //
355
+ // Still exactly one winner: `installBusyReason`, `claimAgentAutoUpdate`
356
+ // and the assignment below are all synchronous, with no await between
357
+ // them, so no second coroutine can interleave into the gap.
358
+ if (this.installBusyReason() !== null)
359
+ return;
360
+ // Claimed before the install and written to disk: a machine that fails for
361
+ // a reason it cannot fix (no space, no network) must go quiet for the day
362
+ // instead of retrying at every session start.
363
+ if (!claimAgentAutoUpdate(runtime.wireKey))
364
+ return;
365
+ this.installInFlight = 'agent_install';
366
+ let outcome;
367
+ try {
368
+ const run = this.opts.installAgent ?? installAgent;
369
+ outcome = await run({ agent: runtime.wireKey, version: latest });
370
+ }
371
+ finally {
372
+ this.installInFlight = null;
373
+ }
374
+ if (outcome.ok) {
375
+ log.info('agent-auto-update: installed', {
376
+ agent: runtime.wireKey,
377
+ from: outcome.fromVersion,
378
+ to: outcome.toVersion,
379
+ });
380
+ }
381
+ else {
382
+ log.warn('agent-auto-update: install failed', {
383
+ agent: runtime.wireKey,
384
+ version: latest,
385
+ error: outcome.detail,
386
+ });
387
+ }
388
+ // After EVERY outcome, for the reason the button's handler gives: a failed
389
+ // install is exactly when the card is most likely to be wrong, because npm
390
+ // can unlink the old package before dying. `reason: 'auto'` is what earns
391
+ // the `dev.server.agent_auto_updated` audit line on the other side (Р14),
392
+ // and `changed` is its payload — omitted when nothing actually moved.
393
+ invalidateAgentVersions();
394
+ const moved = outcome.ok && outcome.toVersion !== undefined && outcome.toVersion !== outcome.fromVersion
395
+ ? { agent: outcome.agent, from: outcome.fromVersion, to: outcome.toVersion }
396
+ : undefined;
397
+ await this.publishAgentVersions('auto', moved);
398
+ }
399
+ catch (error) {
400
+ // `installAgent` never throws, but `measureAgent` spawns processes and the
401
+ // whole point of this path is that nothing it does can reach the session.
402
+ log.warn('agent-auto-update: gave up', {
403
+ agent: runtime.wireKey,
404
+ error: String(error instanceof Error ? error.message : error),
405
+ });
406
+ }
407
+ }
408
+ /** Once a day, per Р12 — the vendor adds at most one file a day. */
409
+ static AGENT_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
410
+ /** Long enough for `hello_ack` to have told us which sessions are live. */
411
+ static AGENT_CLEANUP_FIRST_DELAY_MS = 60 * 60 * 1_000;
412
+ agentCleanupTimer;
413
+ agentCleanupFirstTimer;
414
+ /**
415
+ * Throw away the Claude version files nobody will run again (Р12, §4.7).
416
+ *
417
+ * Gated on NO LIVE AGENT PROCESS — `liveSessionCount`, not `sessions.size`.
418
+ *
419
+ * The stricter reading («nothing tracked at all») was the first version of
420
+ * this gate and it made the feature dead: parked sessions sit in that map
421
+ * until somebody closes them, so on any machine actually in use the sweep
422
+ * would never once have run, and Р12 would have shipped as a no-op nobody
423
+ * noticed. A parked session is not a risk to a version file either — it has
424
+ * no process, and when it wakes it relaunches through the launcher, which is
425
+ * the one file this sweep never touches.
426
+ *
427
+ * What guards a version somebody IS running is the per-file check in
428
+ * `agent-cleanup.ts` (`/proc/<pid>/exe`), which is the plan's own mechanism —
429
+ * «проверка по списку процессов». This gate is the cheap belt beside it: it
430
+ * keeps the sweep away from the minutes when Claude is most likely to be
431
+ * mid-spawn, where the process check has a window it cannot see.
432
+ *
433
+ * Also skipped while an install holds the lock: `claude install <version>` is
434
+ * rewriting the launcher right then, and «which file is current» has no stable
435
+ * answer until it finishes.
436
+ */
437
+ sweepOldAgentVersions() {
438
+ // The empty string matches no session id, so this is «any live process».
439
+ if (this.liveSessionCount('') > 0)
440
+ return;
441
+ if (this.installBusyReason() !== null)
442
+ return;
443
+ try {
444
+ const swept = (this.opts.pruneAgentVersions ?? pruneNativeClaudeVersions)();
445
+ if (swept.removed.length > 0) {
446
+ log.info('agent-cleanup: removed old Claude versions', {
447
+ removed: swept.removed,
448
+ freedMb: Math.round(swept.freedBytes / 1024 / 1024),
449
+ });
450
+ }
451
+ }
452
+ catch (error) {
453
+ log.warn('agent-cleanup: sweep failed', { error: String(error) });
454
+ }
134
455
  }
456
+ /** How often the seat report re-derives the truth. See the constructor. */
457
+ static SLOTS_REPORT_INTERVAL_MS = 15_000;
458
+ slotsTimer;
135
459
  /**
136
460
  * Push every unacked verdict at the API.
137
461
  *
@@ -153,14 +477,90 @@ export class Supervisor {
153
477
  get activeSessionIds() {
154
478
  return [...this.sessions.keys()];
155
479
  }
480
+ /** The last set of seat-holders we told the API about, to send only changes. */
481
+ lastPublishedSlots = '';
482
+ /**
483
+ * Tell the API which sessions are actually holding a seat here (0.46.0).
484
+ *
485
+ * The seat count is a property of THIS process, and until now the API could
486
+ * only guess it from its own rows. The guess was wrong whenever a session
487
+ * ended on paper without releasing its entry, and wrong in the direction that
488
+ * costs a person their session: the dashboard offers a seat, the create
489
+ * passes, this runner refuses, the session dies in fifty milliseconds.
490
+ *
491
+ * Called from `reportStatus` — the choke point every seat change already
492
+ * passes through — and from the interval in the constructor, which is the net
493
+ * under any path that forgets. Both are safe to call as often as they like:
494
+ * this compares what it is about to say against what it last said and returns
495
+ * without touching the socket when nothing moved.
496
+ *
497
+ * Two lists, because two different questions are being asked and answering
498
+ * both with one number is wrong in both directions:
499
+ *
500
+ * - `sessionIds` — everything still tracked here, parked or not. This is the
501
+ * reconciliation list, and it must be generous: a session the API has
502
+ * buried needs laying to rest whatever state it is in on this side.
503
+ * - `blockingIds` — what a new session would really have to wait for.
504
+ * `ensureCapacity` parks an idle REVIEW or WAITING_INPUT session to make
505
+ * room, so those seats are available on demand.
506
+ *
507
+ * Reporting the second as the first would have the API stop live REVIEW
508
+ * sessions; reporting the first as the second would call a machine full while
509
+ * it had room. Both were caught by the independent QA review of this change.
510
+ */
511
+ publishSlots() {
512
+ const tracked = [];
513
+ const blocking = [];
514
+ for (const entry of this.sessions.values()) {
515
+ const id = entry.descriptor.id;
516
+ tracked.push(id);
517
+ if (entry.stopRequested || entry.parkRequested || !entry.session)
518
+ continue;
519
+ // `ensureCapacity` would park this one to make room, so its seat is
520
+ // available on demand and does not refuse anybody. Counting it as taken
521
+ // reported a machine with three finished-turn sessions as full while it
522
+ // would happily have started a fourth (independent QA, 31.08.2026).
523
+ if (this.isParkable(entry))
524
+ continue;
525
+ blocking.push(id);
526
+ }
527
+ tracked.sort();
528
+ blocking.sort();
529
+ const limit = this.maxSessions;
530
+ const fingerprint = `${limit}:${tracked.join(',')}|${blocking.join(',')}`;
531
+ if (fingerprint === this.lastPublishedSlots)
532
+ return;
533
+ // Recorded only when it actually went out. A frame dropped because the
534
+ // socket was down must not be remembered as sent — otherwise the seat
535
+ // report would go quiet until the set changed again, which on an idle
536
+ // machine is exactly never.
537
+ if (this.ws.send({
538
+ type: 'session_slots',
539
+ sessionIds: tracked,
540
+ blockingIds: blocking,
541
+ maxSessions: limit,
542
+ })) {
543
+ this.lastPublishedSlots = fingerprint;
544
+ }
545
+ }
156
546
  async onFrame(frame) {
157
547
  switch (frame.type) {
158
548
  case 'hello_ack':
549
+ // A new connection knows nothing about the seats we reported to the
550
+ // last one — the API keeps that beside the socket, not in the database,
551
+ // because it is only true while the socket is. Forget what we told the
552
+ // old one so the first tick after this reconnect actually sends.
553
+ this.lastPublishedSlots = '';
159
554
  this.setMaxSessions(frame.maxSessions);
160
555
  await this.reconcile(frame.sessions);
556
+ this.publishSlots();
161
557
  // A build that finished while the socket was down has its verdict
162
558
  // sitting on disk. This is the moment it can be delivered.
163
559
  this.flushVerifyReports();
560
+ // The card of a machine that just came back must not show yesterday's
561
+ // agent versions. Deliberately not awaited: this same frame carries the
562
+ // session reconciliation, and a slow `codex doctor` must not hold it up.
563
+ void this.publishAgentVersions('measured');
164
564
  break;
165
565
  case 'verify_report_ack':
166
566
  this.verifyReports.ack(frame.runId);
@@ -325,6 +725,17 @@ export class Supervisor {
325
725
  });
326
726
  return;
327
727
  }
728
+ /**
729
+ * Р13: move THIS session's agent forward, behind this session's back.
730
+ *
731
+ * Deliberately not awaited, and started before the workspace is prepared:
732
+ * the person asked for a session, and a 320 MB download must not be
733
+ * something they wait through. This session then finishes on the file it
734
+ * has already opened — replacing a binary does not disturb a process that
735
+ * holds it — and the new version takes effect for the next one. That is
736
+ * exactly the promise the switch's caption makes on the card.
737
+ */
738
+ void this.maybeAutoUpdateAgent(descriptor);
328
739
  const running = {
329
740
  descriptor,
330
741
  journal: this.journals.open(descriptor.id),
@@ -1111,6 +1522,11 @@ export class Supervisor {
1111
1522
  isTerminal(running.lastReported)) {
1112
1523
  this.journals.closeAndDelete(descriptor.id);
1113
1524
  }
1525
+ // The map just lost an entry, and this is the one place where that happens
1526
+ // WITHOUT a status frame to carry the news: the terminal status was already
1527
+ // reported before the process stream ended. Without this the API goes on
1528
+ // believing the runner tracks a session that is gone, until the next tick.
1529
+ this.publishSlots();
1114
1530
  // A resume arrived while this life was winding down — start it now that the
1115
1531
  // map entry is gone. One place, after every removal path above.
1116
1532
  if (running.pendingRestart && !this.sessions.has(descriptor.id)) {
@@ -1356,7 +1772,12 @@ export class Supervisor {
1356
1772
  });
1357
1773
  }
1358
1774
  else {
1359
- this.reportStatus(descriptor.id, 'FAILED', {
1775
+ // The turn failed for a reason nothing above could soften: not a plan
1776
+ // refusal, not a retryable provider fault. FAILED is terminal, so the
1777
+ // process and the seat go with it — see `finishSession`. Reporting the
1778
+ // status alone is what left this machine holding a seat for a session
1779
+ // the API had already buried (31.08.2026).
1780
+ this.finishSession(running, 'FAILED', {
1360
1781
  costUsd: running.costUsd,
1361
1782
  activeMs: Supervisor.spentMs(running),
1362
1783
  errorMessage: event.errorMessage ?? 'Agent turn failed',
@@ -1620,7 +2041,43 @@ export class Supervisor {
1620
2041
  // No `lastPrompt` guard: a free CHAT session boots with an empty prompt
1621
2042
  // and is exactly the case that hits an auth failure at startup, so
1622
2043
  // requiring one excluded the sessions that need the retry most.
1623
- if (isAuthCode(event.code) && !running.authRetryDone) {
2044
+ // FIRST, and before any decision about retrying (#365). This is the only
2045
+ // authority on a login the credentials file cannot see through — a
2046
+ // provider-side revocation leaves the file looking perfectly healthy,
2047
+ // and `codex login status` never leaves the machine — so it is the one
2048
+ // thing that turns the server panel from «signed in» to «sign in
2049
+ // needed» (#121).
2050
+ //
2051
+ // It used to sit BELOW the retry, i.e. it wanted a SECOND refusal. On
2052
+ // Codex a second refusal never came: the first one was swallowed in
2053
+ // favour of a relaunch that could not happen, so the panel kept showing
2054
+ // a dead login as healthy — and the owner spent an hour signing into
2055
+ // the wrong machine while this line waited for its turn.
2056
+ //
2057
+ // `auth_expired` ONLY, and not every auth code. `auth_missing` means the
2058
+ // credential was locally absent — nothing was refused — and the panel's
2059
+ // own probe already answers «not signed in» for that machine without
2060
+ // any help from here. Marking it would only add a way to be wrong: the
2061
+ // mark outranks a healthy file for fifteen minutes, so a home that was
2062
+ // repaired a second later would keep a red panel over a working login.
2063
+ if (event.code === 'auth_expired') {
2064
+ const refused = relayAgent(String(descriptor.agent).toLowerCase());
2065
+ if (refused)
2066
+ noteAgentAuthFailure(refused);
2067
+ }
2068
+ // Only `auth_missing` is worth a relaunch, and the distinction is the
2069
+ // adapter's own (`probeAuth`), not a guess from prose.
2070
+ //
2071
+ // - `auth_missing` — there is no credential where we look. Usually the
2072
+ // link into the shared store went out from under a live daemon, and
2073
+ // the adapter re-asserts its home on start, so one relaunch really
2074
+ // does fix it.
2075
+ // - `auth_expired` — the provider refused a credential that IS there.
2076
+ // A repeat presents the same dead token and gets the same answer,
2077
+ // while the person waits for a retry that was never going to work.
2078
+ // `error-policy.ts` has said exactly this about Claude since 0.44.1
2079
+ // («A person has to sign in; a repeat cannot») — the two now agree.
2080
+ if (event.code === 'auth_missing' && !running.authRetryDone) {
1624
2081
  running.authRetry = { prompt: running.lastPrompt };
1625
2082
  this.sendEvent(running, 'notice', {
1626
2083
  level: 'warn',
@@ -1628,22 +2085,18 @@ export class Supervisor {
1628
2085
  });
1629
2086
  return;
1630
2087
  }
1631
- // The retry is spent and the agent is still refused — this is the only
1632
- // authority on a login the credentials file cannot see through (a
1633
- // provider-side revocation leaves the file looking perfectly healthy).
1634
- // The panel is told from here, not from a guess (#121).
1635
- if (isAuthCode(event.code)) {
1636
- const refused = relayAgent(String(descriptor.agent).toLowerCase());
1637
- if (refused)
1638
- noteAgentAuthFailure(refused);
1639
- }
1640
2088
  // Forward the code: the API stores the payload as-is, so the dashboard
1641
2089
  // can offer "Sign in" instead of a dead error card.
1642
2090
  this.sendEvent(running, 'error', {
1643
2091
  message: event.message,
1644
2092
  ...(event.code ? { code: event.code } : {}),
1645
2093
  });
1646
- this.reportStatus(descriptor.id, 'FAILED', {
2094
+ // Through `finishSession`, not `reportStatus`: this branch is the one
2095
+ // the 31.08.2026 incident was traced to. It filed the session FAILED
2096
+ // and returned, and the agent process went on streaming for another
2097
+ // seventy-one seconds — holding a seat on a machine whose database said
2098
+ // it was free.
2099
+ this.finishSession(running, 'FAILED', {
1647
2100
  costUsd: running.costUsd,
1648
2101
  activeMs: Supervisor.spentMs(running),
1649
2102
  errorMessage: event.message,
@@ -2185,9 +2638,11 @@ export class Supervisor {
2185
2638
  }
2186
2639
  if (!running.session) {
2187
2640
  // The process died while we waited. Relaunching is `launchAgent`'s job and
2188
- // it needs a prompt; without one there is nothing honest to do here.
2189
- this.clearApiRetry(running);
2190
- this.reportStatus(running.descriptor.id, 'FAILED', {
2641
+ // it needs a prompt; without one there is nothing honest to do here — so
2642
+ // the session ends, entry and all (`finishSession`). It used to end only
2643
+ // on paper, and the row left behind then swallowed the next
2644
+ // `session_start` for the same id.
2645
+ this.finishSession(running, 'FAILED', {
2191
2646
  errorMessage: 'The agent process ended while waiting to retry',
2192
2647
  });
2193
2648
  return;
@@ -2708,6 +3163,65 @@ export class Supervisor {
2708
3163
  });
2709
3164
  }
2710
3165
  }
3166
+ /**
3167
+ * Let go of a session the API says no longer exists, so its files can go.
3168
+ *
3169
+ * `purge_session` and `clean` used to refuse outright while ANY entry for the
3170
+ * id was in the map. That reads as caution and behaves as a deadlock: the
3171
+ * frame arrives precisely because the API has already deleted the row, so
3172
+ * nothing will ever come along to release the entry, the worktree stays on
3173
+ * disk, and the retry runs until the purge ages out of its window. On
3174
+ * production one such tombstone was still failing two hours later, on a
3175
+ * machine that was refusing new sessions for exactly the seat it held.
3176
+ *
3177
+ * The API is the authority on whether a session exists. If it says gone, the
3178
+ * honest answer is to end it here too — the same teardown a reconnect
3179
+ * performs for a session missing from `hello_ack` — and only then report that
3180
+ * we could not finish, if the process really will not go.
3181
+ *
3182
+ * Returns true when nothing is holding the id any more.
3183
+ */
3184
+ async releaseForCleanup(sessionId) {
3185
+ const running = this.sessions.get(sessionId);
3186
+ if (!running)
3187
+ return true;
3188
+ log.warn('supervisor: cleanup asked for a session this runner still held', { sessionId });
3189
+ if (!running.stopRequested) {
3190
+ running.stopRequested = true;
3191
+ this.clearBudgetTimers(running);
3192
+ this.clearApiRetry(running);
3193
+ this.clearEmptyTurn(running);
3194
+ this.withdrawOpenQuestions(running, 'session_stopped');
3195
+ // No status report: the row is already gone on the other side, so a frame
3196
+ // about it would be answered with «Unknown session» and nothing else.
3197
+ running.session?.stop('session_stopped');
3198
+ if (!running.session)
3199
+ this.sessions.delete(sessionId);
3200
+ this.drainSessionsWaitingForCapacity();
3201
+ this.publishSlots();
3202
+ }
3203
+ // `pumpEvents` removes the entry when the process stream actually ends, and
3204
+ // that is asynchronous. Bounded wait rather than an unbounded one: the
3205
+ // caller is a command with its own timeout, and a purge that has to happen
3206
+ // on the next drain instead of this one costs a retry, not the worktree.
3207
+ for (let waited = 0; waited < Supervisor.CLEANUP_RELEASE_MS; waited += 100) {
3208
+ if (!this.sessions.has(sessionId))
3209
+ return true;
3210
+ await new Promise((resolve) => setTimeout(resolve, 100));
3211
+ }
3212
+ return !this.sessions.has(sessionId);
3213
+ }
3214
+ /**
3215
+ * How long cleanup waits for a stopped agent process to actually be gone.
3216
+ *
3217
+ * Deliberately a small slice of the API's ten-second command budget: removing
3218
+ * the worktree still has to happen after this, under the repository lock, and
3219
+ * a purge that times out on the wire is reported as a failure even when it
3220
+ * succeeded here. Three seconds is enough for an ordinary exit; anything
3221
+ * slower is better answered honestly, so the drain comes back and finds the
3222
+ * entry already gone.
3223
+ */
3224
+ static CLEANUP_RELEASE_MS = 3_000;
2711
3225
  stopSession(sessionId) {
2712
3226
  const running = this.sessions.get(sessionId);
2713
3227
  if (!running)
@@ -2726,6 +3240,59 @@ export class Supervisor {
2726
3240
  this.journals.closeAndDelete(sessionId);
2727
3241
  }
2728
3242
  }
3243
+ this.publishSlots();
3244
+ }
3245
+ /**
3246
+ * The ONE way a session ends on this runner.
3247
+ *
3248
+ * A terminal status is two facts, not one: the API is told the session is
3249
+ * over, AND this machine stops holding a seat for it. They used to be
3250
+ * separate lines at eight call sites, and three of them wrote only the first
3251
+ * — `settleTurnStatus`'s failed-turn branch, `forwardEvent`'s `case 'error'`
3252
+ * and `runApiRetry`. Each left a `RunningSession` in the map with a live
3253
+ * `session` handle and `stopRequested === false`, which is precisely what
3254
+ * `liveSessionCount` counts. The seat was then held for a session the API had
3255
+ * already buried, until the next reconnect — weeks, on a healthy runner.
3256
+ *
3257
+ * On production (31.08.2026) that arithmetic refused a fourth session on a
3258
+ * machine whose database said one was running, and the dashboard had offered
3259
+ * the seat a moment earlier. `launchCrashed` had the rule right all along
3260
+ * («an entry left in the map would hold one of the runner's few slots»); it
3261
+ * just could not be the only place that knew it.
3262
+ *
3263
+ * Deliberately NOT folded into `reportStatus`: that method is also how a
3264
+ * session reaches REVIEW, WAITING_INPUT and RUNNING, and a teardown hidden
3265
+ * inside it would be invisible at exactly the call sites that must not tear
3266
+ * anything down. The name says what it does.
3267
+ */
3268
+ finishSession(running, status, extra) {
3269
+ const sessionId = running.descriptor.id;
3270
+ // Before the report, so a frame that races the teardown cannot re-arm
3271
+ // anything the teardown is in the middle of taking away.
3272
+ running.stopRequested = true;
3273
+ this.clearBudgetTimers(running);
3274
+ this.clearApiRetry(running);
3275
+ this.clearEmptyTurn(running);
3276
+ // The cards die with the turn: a question nobody can answer any more is a
3277
+ // worse thing to leave on screen than no question at all.
3278
+ this.withdrawOpenQuestions(running, 'session_stopped');
3279
+ this.reportStatus(sessionId, status, extra);
3280
+ if (running.session) {
3281
+ // `pumpEvents` deletes the entry when the stream ends, and its
3282
+ // `stopRequested` branch skips the STOPPED frame because `lastReported`
3283
+ // is already terminal. Until then the seat is free anyway: `stopRequested`
3284
+ // takes the entry out of `liveSessionCount` on this very line.
3285
+ running.session.stop('session_stopped');
3286
+ }
3287
+ else {
3288
+ this.sessions.delete(sessionId);
3289
+ if (this.ws.connected && running.journal.unacked().length === 0) {
3290
+ this.journals.closeAndDelete(sessionId);
3291
+ }
3292
+ }
3293
+ // A freed seat is only useful to whoever is waiting for one.
3294
+ this.drainSessionsWaitingForCapacity();
3295
+ this.publishSlots();
2729
3296
  }
2730
3297
  // ─── Reconciliation (hello_ack) ────────────────────────────────────
2731
3298
  async reconcile(descriptors) {
@@ -2977,11 +3544,30 @@ export class Supervisor {
2977
3544
  });
2978
3545
  }
2979
3546
  }
2980
- // REVIEW stays REVIEW (WAITING_INPUT is not reachable from it);
2981
- // mid-turn statuses are downgraded to "waiting for the user".
2982
- if (descriptor.status !== 'REVIEW') {
2983
- this.reportStatus(descriptor.id, 'WAITING_INPUT', {});
2984
- }
3547
+ /**
3548
+ * REVIEW stays REVIEW (WAITING_INPUT is not reachable from it);
3549
+ * mid-turn statuses are downgraded to "waiting for the user".
3550
+ *
3551
+ * REVIEW is now REPORTED rather than skipped — ticket #356, and the
3552
+ * news in that frame is not the status, which has not moved. It is
3553
+ * the ZERO riding on it.
3554
+ *
3555
+ * `running.backgroundTasks` is seeded to 0 above, because a runner
3556
+ * restart takes every subagent with it. But the number the API holds
3557
+ * is written by the runner ALONE, and it is only ever written by a
3558
+ * frame that carries the field — which `reportStatus` attaches only
3559
+ * for a tracked session. Skipping the report here left the API
3560
+ * believing whatever the dead process last said, for good: the badge
3561
+ * would keep saying «Agents», the Inbox would keep hiding the card,
3562
+ * and no later frame would ever correct either. `setBackgroundTasks`
3563
+ * cannot help — it compares against the in-memory 0 and returns
3564
+ * early, having nothing to announce.
3565
+ *
3566
+ * Same-status frames are legal (`canDevSessionTransition` answers
3567
+ * `true` for `from === to`), so this costs one no-op write and buys
3568
+ * back a session that would otherwise have been lost.
3569
+ */
3570
+ this.reportStatus(descriptor.id, statusForReport(running), {});
2985
3571
  this.flushPendingMessages(running);
2986
3572
  }
2987
3573
  else if (descriptor.status === 'WAITING_INPUT') {
@@ -3072,8 +3658,22 @@ export class Supervisor {
3072
3658
  const sessionId = frame.sessionId;
3073
3659
  if (!sessionId)
3074
3660
  return void reply({ ok: false, error: 'sessionId is required' });
3075
- const running = this.sessions.get(sessionId);
3076
- if (running)
3661
+ /**
3662
+ * `clean` keeps the flat refusal. `purge_session` does not. The
3663
+ * difference is who is allowed to ask.
3664
+ *
3665
+ * `purge_session` only ever arrives after the API has deleted the
3666
+ * row, so «wait for it to wind down» waits for something that will
3667
+ * never happen — that is the deadlock `releaseForCleanup` exists to
3668
+ * break. `clean` is a user-callable command
3669
+ * (`POST /dev/sessions/:id/commands`, any dev member) and the API
3670
+ * checks no status before relaying it. This refusal is the ONLY guard
3671
+ * on that route: releasing here would stop a mid-turn agent and then
3672
+ * `git worktree remove --force` its worktree, destroying every
3673
+ * uncommitted change in it. Caught by the independent QA review of
3674
+ * this change, by two reviewers separately.
3675
+ */
3676
+ if (this.sessions.get(sessionId))
3077
3677
  return void reply({ ok: false, error: 'Session is still active — stop it first' });
3078
3678
  // `worktree remove` also mutates the shared .git registration.
3079
3679
  await this.withRepoLockFor(sessionWorktreePath(sessionId), () => removeSessionWorktree(sessionId));
@@ -3085,7 +3685,7 @@ export class Supervisor {
3085
3685
  const sessionId = frame.sessionId;
3086
3686
  if (!sessionId)
3087
3687
  return void reply({ ok: false, error: 'sessionId is required' });
3088
- if (this.sessions.get(sessionId)) {
3688
+ if (!(await this.releaseForCleanup(sessionId))) {
3089
3689
  return void reply({ ok: false, error: 'Session is still active — stop it first' });
3090
3690
  }
3091
3691
  const branch = str(frame.args?.['branch']);
@@ -3777,6 +4377,43 @@ export class Supervisor {
3777
4377
  return void reply({ ok: false, error: 'root argument is required' });
3778
4378
  return void reply({ ok: true, result: fsView(root, str(frame.args?.['path']) ?? '.') });
3779
4379
  }
4380
+ /**
4381
+ * Выложить файл с этой машины (0.46.0).
4382
+ *
4383
+ * Адрес, куда уходят байты, раннер собирает САМ из своего `apiUrl` —
4384
+ * во фрейме его нет и быть не должно. Иначе одна подделанная команда
4385
+ * стала бы способом вытянуть файл с чужой машины на чужой хост.
4386
+ */
4387
+ case 'fs_publish': {
4388
+ const root = str(frame.args?.['root']);
4389
+ const filePath = str(frame.args?.['path']);
4390
+ const slotId = str(frame.args?.['slotId']);
4391
+ if (!root || !filePath || !slotId) {
4392
+ return void reply({ ok: false, error: 'root, path and slotId are required' });
4393
+ }
4394
+ if (!this.opts.apiUrl || !this.opts.runnerToken) {
4395
+ return void reply({
4396
+ ok: false,
4397
+ error: 'This runner has no API credentials configured',
4398
+ });
4399
+ }
4400
+ const maxBytesRaw = frame.args?.['maxBytes'];
4401
+ const maxBytes = typeof maxBytesRaw === 'number' ? maxBytesRaw : 0;
4402
+ try {
4403
+ const result = await publishFile({
4404
+ root,
4405
+ relPath: filePath,
4406
+ slotId,
4407
+ maxBytes,
4408
+ apiUrl: this.opts.apiUrl,
4409
+ token: this.opts.runnerToken,
4410
+ });
4411
+ return void reply({ ok: true, result });
4412
+ }
4413
+ catch (error) {
4414
+ return void reply({ ok: false, error: String(error.message ?? error) });
4415
+ }
4416
+ }
3780
4417
  case 'self_update': {
3781
4418
  // Everything that can refuse this lives in self-update.ts; here we
3782
4419
  // only make sure the answer is on the wire BEFORE the process goes
@@ -3791,10 +4428,10 @@ export class Supervisor {
3791
4428
  // Second line of defence behind the API's per-server lock: two
3792
4429
  // overlapping `npm install -g` into the same prefix is not something
3793
4430
  // to leave to chance on someone else's machine.
3794
- if (this.selfUpdateInFlight) {
3795
- return void reply({ ok: false, error: 'An update is already running' });
3796
- }
3797
- this.selfUpdateInFlight = true;
4431
+ const busy = this.installBusyReason();
4432
+ if (busy)
4433
+ return void reply({ ok: false, error: busy });
4434
+ this.installInFlight = 'self_update';
3798
4435
  const run = this.opts.selfUpdate ?? selfUpdate;
3799
4436
  let outcome;
3800
4437
  try {
@@ -3802,7 +4439,7 @@ export class Supervisor {
3802
4439
  }
3803
4440
  catch (error) {
3804
4441
  // A failed update must be retryable without restarting the daemon.
3805
- this.selfUpdateInFlight = false;
4442
+ this.installInFlight = null;
3806
4443
  throw error;
3807
4444
  }
3808
4445
  // Deliberately NOT cleared on success: the process only exits ~1.5s
@@ -3810,7 +4447,7 @@ export class Supervisor {
3810
4447
  // API releases its own lock the moment the reply lands. Clearing here
3811
4448
  // left a window in which a second press started an `npm install -g`
3812
4449
  // that systemd then killed mid-flight (QA-103 MINOR-8).
3813
- this.selfUpdateInFlight = outcome.ok && outcome.restart;
4450
+ this.installInFlight = outcome.ok && outcome.restart ? 'self_update' : null;
3814
4451
  reply({
3815
4452
  ok: outcome.ok,
3816
4453
  result: outcome,
@@ -3820,6 +4457,69 @@ export class Supervisor {
3820
4457
  this.opts.onRestartRequested?.(outcome);
3821
4458
  return;
3822
4459
  }
4460
+ case 'agent_install': {
4461
+ // The veto is enforced here as well as withheld from `hello`: an API
4462
+ // that has not noticed still must not install anything on a machine
4463
+ // whose owner said no.
4464
+ if (this.opts.agentInstallEnabled === false) {
4465
+ return void reply({
4466
+ ok: false,
4467
+ error: 'Installing agents is switched off on this server ([agents] install_enabled = false)',
4468
+ });
4469
+ }
4470
+ // Only these three ever cross the wire. The package name, the
4471
+ // installer URL and the command come from the registry compiled into
4472
+ // this build — a server that could name them would be a server that
4473
+ // could run anything here (§6 of the plan).
4474
+ const agent = str(frame.args?.['agent']);
4475
+ if (!agent)
4476
+ return void reply({ ok: false, error: 'agent is required' });
4477
+ const version = str(frame.args?.['version']);
4478
+ if (!version)
4479
+ return void reply({ ok: false, error: 'version is required' });
4480
+ const allowDowngrade = frame.args?.['allowDowngrade'] === true;
4481
+ const installBusy = this.installBusyReason();
4482
+ if (installBusy)
4483
+ return void reply({ ok: false, error: installBusy });
4484
+ this.installInFlight = 'agent_install';
4485
+ let installed;
4486
+ try {
4487
+ const runInstall = this.opts.installAgent ?? installAgent;
4488
+ installed = await runInstall({
4489
+ agent,
4490
+ version,
4491
+ ...(allowDowngrade ? { allowDowngrade: true } : {}),
4492
+ });
4493
+ }
4494
+ finally {
4495
+ // Always cleared, unlike `self_update`: an agent install does not
4496
+ // restart the daemon, so a holder left behind would block every
4497
+ // later install for the life of the process.
4498
+ this.installInFlight = null;
4499
+ }
4500
+ reply({
4501
+ ok: installed.ok,
4502
+ result: installed,
4503
+ ...(installed.ok ? {} : { error: installed.detail ?? 'Install failed' }),
4504
+ });
4505
+ // Re-measure after EVERY terminal outcome, not only after a success.
4506
+ // A failed install is exactly when the card is most likely to be
4507
+ // wrong: npm can unlink the old global package before dying, and a
4508
+ // rollback can fail too — and the hour-long measurement cache would
4509
+ // then keep re-publishing «0.150.0, up to date» about a machine that
4510
+ // no longer has the agent at all, on every reconnect.
4511
+ //
4512
+ // `changed` stays conditional: it is the audit payload (Р14), and
4513
+ // «updated from 1.2.3 to 1.2.3» would be a line about nothing.
4514
+ invalidateAgentVersions();
4515
+ const moved = installed.ok &&
4516
+ installed.toVersion !== undefined &&
4517
+ installed.toVersion !== installed.fromVersion
4518
+ ? { agent: installed.agent, from: installed.fromVersion, to: installed.toVersion }
4519
+ : undefined;
4520
+ void this.publishAgentVersions('manual', moved);
4521
+ return;
4522
+ }
3823
4523
  /**
3824
4524
  * «Would this file reach the agent, and how big is it» (ticket #192).
3825
4525
  *
@@ -4193,9 +4893,20 @@ export class Supervisor {
4193
4893
  // would mean it goes on claiming background work forever.
4194
4894
  ...(running ? { backgroundTasks: running.backgroundTasks } : {}),
4195
4895
  });
4896
+ // Every seat change is accompanied by a status report — a session starting,
4897
+ // parking, ending. Hooking the seat report here rather than at each of those
4898
+ // is the whole point: «remember to also tell the API» is the rule that had
4899
+ // already been broken in three places, and it is what this frame exists to
4900
+ // stop mattering. `publishSlots` sends only when the answer actually
4901
+ // changed, and the interval in the constructor is the net under both.
4902
+ this.publishSlots();
4196
4903
  }
4197
4904
  /** Graceful daemon shutdown: kill agents, keep sessions resumable server-side. */
4198
4905
  shutdown() {
4906
+ clearInterval(this.slotsTimer);
4907
+ clearInterval(this.agentVersionsTimer);
4908
+ clearTimeout(this.agentCleanupFirstTimer);
4909
+ clearInterval(this.agentCleanupTimer);
4199
4910
  this.authRelay.cancel();
4200
4911
  for (const running of this.sessions.values()) {
4201
4912
  this.clearBudgetTimers(running);
@@ -4380,10 +5091,6 @@ const AGENT_LABELS = { CLAUDE: 'Claude Code', CODEX: 'Codex' };
4380
5091
  function reportsCost(agent) {
4381
5092
  return agent === 'CLAUDE';
4382
5093
  }
4383
- /** Adapter error codes that mean "the sign-in did not work". */
4384
- function isAuthCode(code) {
4385
- return code === 'auth_expired' || code === 'auth_missing';
4386
- }
4387
5094
  /**
4388
5095
  * Does this descriptor point at work that already exists on a branch?
4389
5096
  * If so, silently creating a fresh branch off HEAD would hide the agent's