@songsid/agend 2.3.0 → 2.3.2-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/README.md +1 -1
  2. package/README.zh-TW.md +1 -1
  3. package/dist/authority/authority-state.d.ts +4 -0
  4. package/dist/authority/authority-state.js +16 -10
  5. package/dist/authority/authority-state.js.map +1 -1
  6. package/dist/authority/session-authorizations.d.ts +7 -1
  7. package/dist/authority/session-authorizations.js +6 -4
  8. package/dist/authority/session-authorizations.js.map +1 -1
  9. package/dist/authority/web-authority.d.ts +19 -6
  10. package/dist/authority/web-authority.js +22 -8
  11. package/dist/authority/web-authority.js.map +1 -1
  12. package/dist/backend/muse.d.ts +20 -2
  13. package/dist/backend/muse.js +154 -16
  14. package/dist/backend/muse.js.map +1 -1
  15. package/dist/channel/adapters/discord.d.ts +2 -0
  16. package/dist/channel/adapters/discord.js +33 -3
  17. package/dist/channel/adapters/discord.js.map +1 -1
  18. package/dist/channel/adapters/telegram.d.ts +1 -1
  19. package/dist/channel/adapters/telegram.js +3 -35
  20. package/dist/channel/adapters/telegram.js.map +1 -1
  21. package/dist/channel/mcp-tools.js +5 -5
  22. package/dist/channel/mcp-tools.js.map +1 -1
  23. package/dist/channel/types.d.ts +11 -0
  24. package/dist/cli.js +6 -3
  25. package/dist/cli.js.map +1 -1
  26. package/dist/config-validator.js +42 -5
  27. package/dist/config-validator.js.map +1 -1
  28. package/dist/daemon.d.ts +47 -3
  29. package/dist/daemon.js +380 -60
  30. package/dist/daemon.js.map +1 -1
  31. package/dist/delivery-outbox.d.ts +23 -1
  32. package/dist/delivery-outbox.js +114 -17
  33. package/dist/delivery-outbox.js.map +1 -1
  34. package/dist/doctor.d.ts +3 -0
  35. package/dist/doctor.js +9 -0
  36. package/dist/doctor.js.map +1 -1
  37. package/dist/event-log.d.ts +5 -0
  38. package/dist/event-log.js +13 -0
  39. package/dist/event-log.js.map +1 -1
  40. package/dist/fleet-context.d.ts +1 -1
  41. package/dist/fleet-lock.d.ts +6 -0
  42. package/dist/fleet-lock.js +19 -0
  43. package/dist/fleet-lock.js.map +1 -1
  44. package/dist/fleet-manager.d.ts +174 -2
  45. package/dist/fleet-manager.js +678 -68
  46. package/dist/fleet-manager.js.map +1 -1
  47. package/dist/fleet-process-handlers.d.ts +1 -0
  48. package/dist/fleet-process-handlers.js +9 -1
  49. package/dist/fleet-process-handlers.js.map +1 -1
  50. package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +11 -0
  51. package/dist/general-knowledge/skills/web-appearance/SKILL.canvas-worker-off.md +4 -4
  52. package/dist/general-knowledge/skills/web-appearance/SKILL.md +7 -7
  53. package/dist/general-knowledge/skills/web-dashboard/SKILL.canvas-worker-off.md +109 -57
  54. package/dist/general-knowledge/skills/web-dashboard/SKILL.md +110 -58
  55. package/dist/guest-seen-people.d.ts +38 -0
  56. package/dist/guest-seen-people.js +78 -0
  57. package/dist/guest-seen-people.js.map +1 -0
  58. package/dist/instance-config-impact.js +4 -1
  59. package/dist/instance-config-impact.js.map +1 -1
  60. package/dist/instance-lifecycle.d.ts +2 -0
  61. package/dist/instance-lifecycle.js +27 -1
  62. package/dist/instance-lifecycle.js.map +1 -1
  63. package/dist/locale.js +62 -12
  64. package/dist/locale.js.map +1 -1
  65. package/dist/log-service.js +1 -0
  66. package/dist/log-service.js.map +1 -1
  67. package/dist/mac-auto-restore.d.ts +117 -0
  68. package/dist/mac-auto-restore.js +294 -0
  69. package/dist/mac-auto-restore.js.map +1 -0
  70. package/dist/needs-you.js +1 -0
  71. package/dist/needs-you.js.map +1 -1
  72. package/dist/outbound-handlers.d.ts +2 -2
  73. package/dist/outbound-handlers.js +5 -5
  74. package/dist/outbound-handlers.js.map +1 -1
  75. package/dist/outbound-schemas.d.ts +2 -0
  76. package/dist/outbound-schemas.js +2 -0
  77. package/dist/outbound-schemas.js.map +1 -1
  78. package/dist/public-web-gateway.js +9 -1
  79. package/dist/public-web-gateway.js.map +1 -1
  80. package/dist/search/search-index.d.ts +10 -2
  81. package/dist/search/search-index.js +17 -4
  82. package/dist/search/search-index.js.map +1 -1
  83. package/dist/service-installer.d.ts +23 -1
  84. package/dist/service-installer.js +40 -11
  85. package/dist/service-installer.js.map +1 -1
  86. package/dist/session-oom-stop.d.ts +45 -0
  87. package/dist/session-oom-stop.js +259 -0
  88. package/dist/session-oom-stop.js.map +1 -0
  89. package/dist/session-oom.d.ts +13 -0
  90. package/dist/session-oom.js +119 -7
  91. package/dist/session-oom.js.map +1 -1
  92. package/dist/settings-api.d.ts +27 -1
  93. package/dist/settings-api.js +108 -2
  94. package/dist/settings-api.js.map +1 -1
  95. package/dist/settings-change.js +37 -3
  96. package/dist/settings-change.js.map +1 -1
  97. package/dist/settings-effect.js +5 -2
  98. package/dist/settings-effect.js.map +1 -1
  99. package/dist/spawn-gate.d.ts +2 -0
  100. package/dist/spawn-gate.js +17 -4
  101. package/dist/spawn-gate.js.map +1 -1
  102. package/dist/start-attempt.d.ts +15 -1
  103. package/dist/start-attempt.js +35 -5
  104. package/dist/start-attempt.js.map +1 -1
  105. package/dist/start-failure-chat.d.ts +41 -0
  106. package/dist/start-failure-chat.js +159 -0
  107. package/dist/start-failure-chat.js.map +1 -0
  108. package/dist/start-failure-notice.d.ts +4 -0
  109. package/dist/start-failure-notice.js +8 -0
  110. package/dist/start-failure-notice.js.map +1 -1
  111. package/dist/start-failure.d.ts +49 -0
  112. package/dist/start-failure.js +141 -1
  113. package/dist/start-failure.js.map +1 -1
  114. package/dist/telegram-api-root.d.ts +1 -0
  115. package/dist/telegram-api-root.js +40 -0
  116. package/dist/telegram-api-root.js.map +1 -0
  117. package/dist/turn-reply-guard.d.ts +25 -0
  118. package/dist/turn-reply-guard.js +111 -12
  119. package/dist/turn-reply-guard.js.map +1 -1
  120. package/dist/types.d.ts +21 -3
  121. package/dist/ui/backend-signin.js +5 -2
  122. package/dist/ui/chat-ambient.js +1 -1
  123. package/dist/ui/chat-effects.js +4 -4
  124. package/dist/ui/chat-render.js +5 -2
  125. package/dist/ui/chat-strings.js +2 -2
  126. package/dist/ui/chat-thread.js +2 -2
  127. package/dist/ui/first-run.js +2 -2
  128. package/dist/ui/fleet-canvas-view.js +114 -0
  129. package/dist/ui/fleet-canvas.js +15 -77
  130. package/dist/ui/fleet-office-editor.js +80 -5
  131. package/dist/ui/fleet-office-layout.js +197 -0
  132. package/dist/ui/fleet-office-model.js +6 -3
  133. package/dist/ui/fleet-office-pack.d.ts +39 -0
  134. package/dist/ui/fleet-office-pack.js +240 -29
  135. package/dist/ui/fleet-office-skin.js +130 -21
  136. package/dist/ui/fleet-office-store.js +75 -4
  137. package/dist/ui/fleet-office-view.js +154 -0
  138. package/dist/ui/fleet-office-zones.js +95 -33
  139. package/dist/ui/fleet-office.js +530 -94
  140. package/dist/ui/fleet-usage.js +7 -6
  141. package/dist/ui/panel-chat.js +20 -7
  142. package/dist/ui/panel-details.js +2 -2
  143. package/dist/ui/panel-fleet.js +11 -77
  144. package/dist/ui/panel-log.js +5 -5
  145. package/dist/ui/panel-settings.js +586 -164
  146. package/dist/ui/search-dialog.js +26 -7
  147. package/dist/ui/settings-appearance.js +82 -29
  148. package/dist/ui/settings-apply.js +42 -7
  149. package/dist/ui/settings-dialogs.js +68 -22
  150. package/dist/ui/settings-draft.js +51 -0
  151. package/dist/ui/settings-guests.js +244 -78
  152. package/dist/ui/settings-learn.js +2 -2
  153. package/dist/ui/settings-model.js +2 -0
  154. package/dist/ui/settings-registry.js +142 -0
  155. package/dist/ui/settings-strings.js +108 -44
  156. package/dist/ui/settings-web-authority.js +34 -23
  157. package/dist/ui/shared/app-i18n.js +14 -0
  158. package/dist/ui/shared/app-nav.js +10 -5
  159. package/dist/ui/shared/app-prefs.js +80 -2
  160. package/dist/ui/shared/app-route.js +39 -5
  161. package/dist/ui/shared/app-shell.js +80 -8
  162. package/dist/ui/shared/app-stream.js +2 -1
  163. package/dist/ui/shared/app.css +251 -20
  164. package/dist/ui/shared/app.js +3 -0
  165. package/dist/ui/shared/effect-stock-trust.js +1 -1
  166. package/dist/ui/shared/identity-color.js +4 -4
  167. package/dist/ui/shared/signin.js +2 -2
  168. package/dist/ui/shared/ui-dialog.js +6 -1
  169. package/dist/ui/shared/ui-icons.js +2 -0
  170. package/dist/ui/shared/usage-dialog.js +21 -11
  171. package/dist/ui/shared/view-strings.js +2 -0
  172. package/dist/ui/shared/web-authorize.js +13 -12
  173. package/dist/web-api.d.ts +9 -3
  174. package/dist/web-api.js +161 -6
  175. package/dist/web-api.js.map +1 -1
  176. package/dist/web-auth.js +7 -1
  177. package/dist/web-auth.js.map +1 -1
  178. package/dist/web-experimental.d.ts +1 -1
  179. package/dist/web-experimental.js +1 -1
  180. package/dist/web-experimental.js.map +1 -1
  181. package/dist/web-mod-agent.js +2 -2
  182. package/dist/web-mod-agent.js.map +1 -1
  183. package/dist/web-office-skins.d.ts +108 -0
  184. package/dist/web-office-skins.js +298 -0
  185. package/dist/web-office-skins.js.map +1 -0
  186. package/dist/web-office-store.d.ts +8 -2
  187. package/dist/web-office-store.js +37 -7
  188. package/dist/web-office-store.js.map +1 -1
  189. package/dist/web-settings-patch.d.ts +6 -0
  190. package/dist/web-settings-patch.js +61 -4
  191. package/dist/web-settings-patch.js.map +1 -1
  192. package/dist/web-shell-routes.d.ts +4 -2
  193. package/dist/web-shell-routes.js +17 -7
  194. package/dist/web-shell-routes.js.map +1 -1
  195. package/dist/working-recovery.d.ts +104 -0
  196. package/dist/working-recovery.js +263 -0
  197. package/dist/working-recovery.js.map +1 -0
  198. package/dist/working-state-journal.d.ts +70 -0
  199. package/dist/working-state-journal.js +106 -0
  200. package/dist/working-state-journal.js.map +1 -0
  201. package/package.json +1 -1
@@ -1,3 +1,4 @@
1
+ import { WorkingRecoveryCoordinator, WORKING_RECOVERY_KIND, buildWorkingRecoveryPrompt } from "./working-recovery.js";
1
2
  import { PUBLIC_TUNNEL_TOKEN_ENV, PUBLIC_TUNNEL_TOKEN_HINT, normalizePublicTunnelToken } from "./public-tunnel-config.js";
2
3
  import { humanOwnerObjectId, webHumanOrigin } from "./human-delivery-receipts.js";
3
4
  import { SettingsBaselines } from "./settings-baseline.js";
@@ -158,6 +159,7 @@ import { agendVersion } from "./agent-whats-new.js";
158
159
  import { OwnerCssStore } from "./web-owner-css-store.js";
159
160
  import { ThemeStore } from "./web-theme-store.js";
160
161
  import { OfficeStore, OfficeWriteLimiter } from "./web-office-store.js";
162
+ import { OfficeSkinStore } from "./web-office-skins.js";
161
163
  import { WebAppearance } from "./web-appearance.js";
162
164
  import { WebEffectLibrary } from "./web-effect-library.js";
163
165
  import { WebEffectTrials } from "./web-effect-trials.js";
@@ -217,6 +219,8 @@ import { assertExplicitInstanceRemoval } from "./instance-removal.js";
217
219
  import { KNOWN_BACKENDS, validateFleetConfig } from "./config-validator.js";
218
220
  import { isRemovedBackend, removedBackendMessage } from "./backend/removed.js";
219
221
  import { presentationState, interactionSummary, sameInteractionOwner } from "./interaction-observation.js";
222
+ import { describeSelfOom } from "./session-oom.js";
223
+ import { assertNoSessionOomStop, readSessionOomStop, captureSessionOomRetry, clearSessionOomStopForRetry, SessionOomStoppedError, sessionOomStopHint } from "./session-oom-stop.js";
220
224
  import { idleGateTimeoutLog } from "./idle-gate-log.js";
221
225
  import { DEFAULT_STUCK_TIMEOUT_MS, readLastInboundAt } from "./daemon.js";
222
226
  import { clearPausedMarker, readPausedAt, readPauseReason, writePausedMarker } from "./pause-marker.js";
@@ -460,6 +464,7 @@ const EFFORT_SELECT_CALLBACK_PREFIX = "effort-select:";
460
464
  const BACKEND_SELECT_CALLBACK_PREFIX = "backend-select:";
461
465
  const INTERACTIVE_ASSIST_CALLBACK_PREFIX = "interactive-assist:";
462
466
  const EXIT_RESTART_CALLBACK_PREFIX = "exit-restart:";
467
+ const OOM_RETRY_CALLBACK_PREFIX = "oom-retry:";
463
468
  /** #1386: how long a paused instance's marker reason is reused before it is read again (transitions drop it at once). */
464
469
  const NEEDS_PAUSE_TTL_MS = 10_000;
465
470
  const HANG_CALLBACK_PREFIX = "hang:";
@@ -471,7 +476,7 @@ const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
471
476
  * group's approval, tips, and the per-user /model and /effort menus (which are not nonce entries at all).
472
477
  */
473
478
  const WEB_MIRRORED_PROMPT_PREFIXES = new Set([
474
- HANG_CALLBACK_PREFIX, EXIT_RESTART_CALLBACK_PREFIX, INTERACTIVE_ASSIST_CALLBACK_PREFIX,
479
+ HANG_CALLBACK_PREFIX, EXIT_RESTART_CALLBACK_PREFIX, OOM_RETRY_CALLBACK_PREFIX, INTERACTIVE_ASSIST_CALLBACK_PREFIX,
475
480
  ]);
476
481
  /**
477
482
  * Where a reply goes on a fleet with no chat platform (web track C4): it is "sent" by being shown in the
@@ -602,6 +607,89 @@ export class FleetManager {
602
607
  managerBootId = randomUUID();
603
608
  deliveryOutbox = null;
604
609
  deliveryOutboxRecovered = false;
610
+ workingRecovery = null;
611
+ workingRecoveryTimer = null;
612
+ workingRecoveryCause = "unknown_interruption";
613
+ recoveryEnabled(name) {
614
+ const own = this.fleetConfig?.instances[name]?.resume_interrupted_work;
615
+ return own ?? this.fleetConfig?.defaults?.resume_interrupted_work ?? true;
616
+ }
617
+ initializeWorkingRecovery(outbox) {
618
+ const store = outbox.workingRecovery;
619
+ this.workingRecoveryCause = store.beginRun(this.managerBootId, process.pid, Date.now());
620
+ this.workingRecovery = new WorkingRecoveryCoordinator({
621
+ store, now: () => performance.now(), active: () => this.startupComplete && !this.shuttingDown && outbox.isOpen,
622
+ pressure: () => {
623
+ const p = this.memoryPressure.snapshot();
624
+ return p.sampledAt === null || Date.now() - p.sampledAt > 60_000 ? { ...p, level: "unknown" } : p;
625
+ }, storm: () => this.stormWindow.isActive(),
626
+ enabled: name => this.recoveryEnabled(name),
627
+ live: name => {
628
+ const daemon = this.daemons.get(name);
629
+ if (this.sessionOomStopped(name))
630
+ return { owner: "", configuration: "", sessionId: null, idle: false,
631
+ paused: false, stopped: true, legacyPending: false, legacyCovered: false, nativeWork: false, cancelled: false, background: null };
632
+ return daemon?.getWorkingRecoveryState?.() ?? null;
633
+ },
634
+ card: id => this.scheduler?.db.board.findCard(id) ?? (this.scheduler ? undefined : null),
635
+ laneFree: name => !this.isInstanceRestarting(name) && !this.activeDurableTargets.has(name)
636
+ && outbox.countForTarget(name, ["queued", "retry_wait", "delivering", "submission_started", "reconciliation_pending"]) === 0,
637
+ admit: (episode, tasks, advisory) => outbox.admitWorkingRecovery(episode, buildWorkingRecoveryPrompt(tasks, this.workingRecoveryCause, advisory), advisory),
638
+ summary: (group, rows) => {
639
+ const general = this.findGeneralInstance();
640
+ if (!general || !this.daemons.has(general))
641
+ return;
642
+ // Our own informational notice must not supersede General's recovery
643
+ // as unrelated pending work before it crosses the input boundary.
644
+ const generalEpisode = store.get(general);
645
+ if (generalEpisode?.outcome === "admitted" && generalEpisode.deliveryId) {
646
+ const state = outbox.get(generalEpisode.deliveryId)?.state;
647
+ if (!state || !["delivered", "uncertain", "failed", "cancelled"].includes(state))
648
+ return;
649
+ }
650
+ const deferredAdvisories = rows.filter(row => row.outcome === "deferred"
651
+ && (this.daemons.get(row.instance)?.getWorkingRecoveryState?.().background ?? null) === null)
652
+ .map(row => `${row.instance} (${row.reason})`);
653
+ store.summary(group, () => outbox.admitSystemNotice(general, `working-recovery-summary:${group}`, `[system:working-recovery-summary] Observed cause: unknown interruption${this.workingRecoveryCause === "unknown_interruption" ? "" : `; prior requested action: ${this.workingRecoveryCause}`}. ${rows.length} interrupted instances; continuation queued ${rows.filter(r => r.outcome === "admitted").length}, excluded ${rows.filter(r => r.outcome === "excluded").length}, still deferred ${rows.filter(r => r.outcome === "deferred").length}. ${rows.filter(r => r.reason === "advisory_background_unknown").length ? `Advisory, background unknown: ${rows.filter(r => r.reason === "advisory_background_unknown").map(r => r.instance).join(", ")}. ` : ""}${deferredAdvisories.length ? `Deferred advisory candidates (background unknown; all other gates still apply): ${deferredAdvisories.join(", ")}. ` : ""}Details are in the Log. Deferred work remains eligible; no action or confirmation is requested.`));
654
+ },
655
+ log: (episode, outcome, reason) => {
656
+ const requestedAction = this.workingRecoveryCause === "unknown_interruption" ? null : this.workingRecoveryCause;
657
+ this.logger.info({ name: episode.instance, episode: episode.key, outcome, reason,
658
+ observedCause: "unknown_interruption", requestedAction }, "Interrupted work recovery decision");
659
+ this.eventLog?.insert(episode.instance, "working_state", {
660
+ message: `Interrupted work: ${outcome} (${reason}); observed cause unknown_interruption${requestedAction ? `; prior requested action ${requestedAction}` : ""}.`,
661
+ reason, outcome, observedCause: "unknown_interruption", requestedAction
662
+ });
663
+ },
664
+ });
665
+ if (this.workingRecoveryTimer)
666
+ clearInterval(this.workingRecoveryTimer);
667
+ this.workingRecoveryTimer = setInterval(() => {
668
+ if (!outbox.isOpen || this.shuttingDown)
669
+ return;
670
+ try {
671
+ this.workingRecovery?.tick();
672
+ this.scheduleDeliveryOutboxPump();
673
+ }
674
+ catch {
675
+ this.logger.warn("Working recovery deferred: prerequisites or durable store unavailable");
676
+ }
677
+ }, 1_000);
678
+ this.workingRecoveryTimer.unref?.();
679
+ }
680
+ recoveryDeliveryEligible(row, writing = false) {
681
+ if (row.kind !== WORKING_RECOVERY_KIND)
682
+ return true;
683
+ const outbox = this.deliveryOutbox, episode = outbox?.workingRecovery.get(row.targetInstance);
684
+ if (!episode || episode.deliveryId !== row.deliveryId
685
+ || outbox.hasOtherPendingWork(row.targetInstance, row.deliveryId)
686
+ || this.workingRecovery?.verdict(episode).outcome === "excluded") {
687
+ outbox?.withdrawWorkingRecovery(row.deliveryId, "recovery superseded or no longer authorized before input");
688
+ return false;
689
+ }
690
+ return (writing ? this.workingRecovery?.accepts(row.targetInstance, episode.key)
691
+ : this.workingRecovery?.canDispatch(row.targetInstance, episode.key)) === true;
692
+ }
605
693
  /** Phase 2b: wakes paused targets for queued durable work (delivery_worker ≠ off). */
606
694
  wakeCoordinator = null;
607
695
  /**
@@ -759,6 +847,8 @@ export class FleetManager {
759
847
  * backend outage during a post-update herd). instanceName → pending retry.
760
848
  */
761
849
  startupRetries = new Map();
850
+ /** #2484: instances whose automatic startup retry is running now (between its timer and its outcome). */
851
+ startupRetriesRunning = new Set();
762
852
  /** Bumped by every explicit stop (incl. the stop half of a restart); fences the outage hand-off. */
763
853
  explicitStopGeneration = new Map();
764
854
  /** instanceName → outage hand-off currently executing; a repeat joins it. */
@@ -1049,7 +1139,12 @@ export class FleetManager {
1049
1139
  }
1050
1140
  officeStoreInstance;
1051
1141
  /** Fleet → Org chart → Office: the offices people made (web-offices.json, beside web-themes.json). */
1052
- get officeStore() { return (this.officeStoreInstance ??= new OfficeStore(this.dataDir)); }
1142
+ get officeStore() { return (this.officeStoreInstance ??= new OfficeStore(this.dataDir, id => this.officeSkins.pack(id))); }
1143
+ officeSkinsInstance;
1144
+ /** #2513: the Office view's uploaded skin packs (<AGEND_HOME>/web-office-skins/). */
1145
+ get officeSkins() { return (this.officeSkinsInstance ??= new OfficeSkinStore(this.dataDir)); }
1146
+ /** How often a person may upload or delete a skin pack; memory only, as the offices' limiter. */
1147
+ officeSkinWrites = new OfficeWriteLimiter(undefined, { perWindow: 10, windowMs: 60_000 });
1053
1148
  /** How often a person may save the offices (PUT /ui/offices); memory only, as the appearance limiter. */
1054
1149
  officeWrites = new OfficeWriteLimiter();
1055
1150
  /** #1629: the theme tools, a person's reset and lock, and the one appearance rate limiter (shared with effects). */
@@ -1792,7 +1887,7 @@ export class FleetManager {
1792
1887
  ...(web && by.guest ? { web_guest: "1" } : {}), // #1920: "[user:… via web (guest), id:…]", as /ui/send
1793
1888
  ...reactions.meta,
1794
1889
  },
1795
- }, by.humanOrigin, button.label)) !== false;
1890
+ }, by.humanOrigin, button.label, this.startRetryHoldFor(set.instance, inbound))) !== false;
1796
1891
  }
1797
1892
  finally {
1798
1893
  // Both false and throw reopen the choice in the controller. Finish the
@@ -1841,8 +1936,19 @@ export class FleetManager {
1841
1936
  }
1842
1937
  this.deliveryOutbox = outbox;
1843
1938
  this.deliveryOutboxRecovered = true;
1939
+ this.initializeWorkingRecovery(outbox);
1940
+ outbox.on("working_references", (names) => {
1941
+ for (const name of names)
1942
+ this.observeWorkingState(name, "obligations");
1943
+ });
1844
1944
  outbox.on("admitted", () => { this.scheduleDeliveryOutboxPump(); this.wakeCoordinator?.kick(); });
1845
1945
  outbox.on("state", (event) => {
1946
+ if (event.deliveryId) {
1947
+ const row = outbox.get(event.deliveryId);
1948
+ if (row)
1949
+ for (const name of new Set([row.sourceInstance, row.targetInstance]))
1950
+ this.observeWorkingState(name, "obligations");
1951
+ }
1846
1952
  this.scheduleDeliveryOutboxPump();
1847
1953
  this.wakeCoordinator?.kick();
1848
1954
  if (event.deliveryId && event.state === "delivered") {
@@ -2001,8 +2107,19 @@ export class FleetManager {
2001
2107
  : item),
2002
2108
  };
2003
2109
  }
2110
+ /** A terminal OOM stop is not an auto-wakeable pause, including after a fleet restart. */
2111
+ sessionOomStopped(name) {
2112
+ return this.daemons.get(name)?.isSessionOomStopped?.() === true || readSessionOomStop(this.getInstanceDir(name)).blocked;
2113
+ }
2114
+ sessionOomStopReason(name) {
2115
+ const state = readSessionOomStop(this.getInstanceDir(name));
2116
+ return state.blocked ? sessionOomStopHint(state)
2117
+ : this.daemons.get(name)?.notAcceptingReason?.() ?? sessionOomStopHint();
2118
+ }
2004
2119
  /** #1641: why a target is not taking queued work right now, or null when nothing known holds it. */
2005
2120
  deliveryWaitingReason(target) {
2121
+ if (this.sessionOomStopped(target))
2122
+ return this.sessionOomStopReason(target);
2006
2123
  if (this.lifecycle.isPaused(target))
2007
2124
  return t("delivery.hold_state", "paused");
2008
2125
  return this.daemons.get(target)?.notAcceptingReason?.() ?? null;
@@ -2015,6 +2132,9 @@ export class FleetManager {
2015
2132
  }
2016
2133
  /** Called after a replacement Daemon object is installed in lifecycle.daemons. */
2017
2134
  onDaemonReady(name, daemonBootId) {
2135
+ const daemon = this.daemons.get(name);
2136
+ if (daemon?.bootId === daemonBootId)
2137
+ this.listenForWorkingState(name, daemon);
2018
2138
  const outbox = this.deliveryOutbox;
2019
2139
  if (!outbox)
2020
2140
  return;
@@ -2038,6 +2158,86 @@ export class FleetManager {
2038
2158
  }
2039
2159
  this.scheduleDeliveryOutboxPump();
2040
2160
  }
2161
+ workingStateListeners;
2162
+ workingStateErrors;
2163
+ /** Attach only to the published owner. Nothing here sends or resumes work. */
2164
+ listenForWorkingState(name, daemon) {
2165
+ if (typeof daemon.getWorkingStateObservation !== "function")
2166
+ return;
2167
+ this.workingStateListeners ??= new WeakSet();
2168
+ if (this.workingStateListeners.has(daemon))
2169
+ return;
2170
+ this.workingStateListeners.add(daemon);
2171
+ daemon.on("working_state", (event) => {
2172
+ try {
2173
+ if (this.daemons.get(name) !== daemon || !sameInteractionOwner(event.observation.owner, daemon.getInteractionSnapshot().owner))
2174
+ return;
2175
+ this.observeWorkingState(name, event.reason, event.observation);
2176
+ }
2177
+ catch { /* A malformed/stale observation cannot affect the daemon. */ }
2178
+ });
2179
+ this.observeWorkingState(name, "published");
2180
+ }
2181
+ observeWorkingState(name, reason, supplied) {
2182
+ const outbox = this.deliveryOutbox, daemon = this.daemons.get(name);
2183
+ if (!outbox || !daemon || typeof daemon.getWorkingStateObservation !== "function")
2184
+ return;
2185
+ try {
2186
+ const observation = supplied ?? daemon.getWorkingStateObservation();
2187
+ if (!sameInteractionOwner(observation.owner, daemon.getInteractionSnapshot().owner))
2188
+ return;
2189
+ const references = [];
2190
+ const unavailable = [];
2191
+ let totalReferences = 0;
2192
+ const board = this.scheduler?.db.board;
2193
+ if (!board)
2194
+ unavailable.push("tasks");
2195
+ else
2196
+ try {
2197
+ const tasks = board.listCards({ assignee: name, status: ["claimed", "blocked", "review"], limit: 16 });
2198
+ totalReferences += tasks.total;
2199
+ for (const card of tasks.cards)
2200
+ references.push({ kind: "task", id: card.uuid, version: card.version,
2201
+ status: card.status });
2202
+ }
2203
+ catch {
2204
+ unavailable.push("tasks");
2205
+ }
2206
+ try {
2207
+ const obligations = outbox.workingObligations(name);
2208
+ references.push(...obligations.references);
2209
+ totalReferences += obligations.total;
2210
+ }
2211
+ catch {
2212
+ unavailable.push("obligations");
2213
+ }
2214
+ const entry = outbox.observeWorkingState(name, { version: 1, reason, observedAt: Date.now(), observation,
2215
+ lastActivityAt: this.lastActivity.get(name) ?? null, references, totalReferences, unavailable });
2216
+ if (!entry)
2217
+ return;
2218
+ if (reason === "cancelled")
2219
+ outbox.workingRecovery.cancel(name, observation.configuration, observation.sessionId);
2220
+ if (entry.prior)
2221
+ outbox.workingRecovery.discover(name, entry.prior);
2222
+ const state = entry.current.observation.executionState ?? "unknown";
2223
+ const counts = { tasks: entry.current.references.filter(r => r.kind === "task").length,
2224
+ owed: entry.current.references.filter(r => r.kind === "owed").length,
2225
+ waiting: entry.current.references.filter(r => r.kind === "waiting").length,
2226
+ background: entry.current.observation.backgroundWork,
2227
+ total: entry.current.totalReferences, unavailable: entry.current.unavailable };
2228
+ this.logger.info({ name, state, reason, revision: entry.revision, ...counts }, "Working-state journal observed");
2229
+ this.eventLog?.insert(name, "working_state", { state, reason, revision: entry.revision, ...counts,
2230
+ message: `Observed ${state}; tasks ${counts.tasks}, replies owed ${counts.owed}, waiting ${counts.waiting}; ${counts.total} references${counts.background === true ? "; background work running" : ""}${unavailable.length ? "; some sources unavailable" : ""}. Journal only; no recovery action.` });
2231
+ this.emitSseEvent("steps_tick", { instance: name });
2232
+ }
2233
+ catch {
2234
+ this.workingStateErrors ??= new Set();
2235
+ if (!this.workingStateErrors.has(name)) {
2236
+ this.workingStateErrors.add(name);
2237
+ this.logger.warn({ name }, "Could not persist working-state observation; delivery and restart behavior unchanged");
2238
+ }
2239
+ }
2240
+ }
2041
2241
  /**
2042
2242
  * #926: the two reply-obligation safety nets. Idle owners with an unanswered
2043
2243
  * request get one reminder (after the grace); requesters get one overdue
@@ -2112,7 +2312,7 @@ export class FleetManager {
2112
2312
  // Phase 2c: a lane a queue worker owns is never claimed here.
2113
2313
  target => this.queueWorkers.has(target) || this.isInstanceRestarting(target) || this.wakeCoordinator?.blocksClaim(target)
2114
2314
  ? null
2115
- : this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets);
2315
+ : this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets, row => this.recoveryDeliveryEligible(row));
2116
2316
  if (!claimed)
2117
2317
  break;
2118
2318
  this.activeDurableTargets.add(claimed.targetInstance);
@@ -2174,7 +2374,7 @@ export class FleetManager {
2174
2374
  available: () => !this.shuttingDown && this.deliveryOutbox?.isOpen === true,
2175
2375
  blocked: () => this.isInstanceRestarting(target) || this.wakeCoordinator?.blocksClaim(target) === true,
2176
2376
  daemonBootId: () => this.daemons.get(target)?.bootId ?? null,
2177
- claim: bootId => this.deliveryOutbox?.claimNext(this.managerBootId, candidate => candidate === target ? bootId : null, new Set()),
2377
+ claim: bootId => this.deliveryOutbox?.claimNext(this.managerBootId, candidate => candidate === target ? bootId : null, new Set(), row => this.recoveryDeliveryEligible(row)),
2178
2378
  tryAcquireBudget: () => {
2179
2379
  if (this.activeDurableTargets.size >= 8)
2180
2380
  return false;
@@ -2215,7 +2415,24 @@ export class FleetManager {
2215
2415
  // including time spent waiting for a handoff/idle gate to return.
2216
2416
  try {
2217
2417
  let sent;
2218
- if (claimed.kind === "raw_paste") {
2418
+ if (claimed.kind === WORKING_RECOVERY_KIND) {
2419
+ const daemon = this.daemons.get(target), key = claimed.sourceKey.replace(/^working-recovery:/, "");
2420
+ if (!daemon || typeof payload.content !== "string")
2421
+ throw new Error("Working recovery owner unavailable");
2422
+ const release = this.workingRecovery?.reserveDispatch(target, key);
2423
+ if (!release) {
2424
+ outbox.retryBeforeBegin(claimed.deliveryId, daemon.bootId, claimed.attemptNo, "working recovery pace or prerequisites pending", 5_000);
2425
+ return;
2426
+ }
2427
+ try {
2428
+ await daemon.deliverWorkingRecovery(payload.content, claimed, () => this.daemons.get(target) === daemon && this.recoveryDeliveryEligible(claimed, true));
2429
+ }
2430
+ finally {
2431
+ release();
2432
+ }
2433
+ sent = true;
2434
+ }
2435
+ else if (claimed.kind === "raw_paste") {
2219
2436
  if (payload.type !== "raw_paste" || typeof payload.content !== "string") {
2220
2437
  throw new Error("Durable raw_paste row has an invalid payload");
2221
2438
  }
@@ -2899,6 +3116,7 @@ export class FleetManager {
2899
3116
  }
2900
3117
  /** SIGUSR1: full process reload. Names who asked (#1120). */
2901
3118
  onFullRestartSignal() {
3119
+ this.recordWorkingRestartIntent("manual_restart");
2902
3120
  this.logger.info(`Received SIGUSR1, initiating full restart (process reload)... ${describeSignalSource(this.dataDir, "SIGUSR1")}`);
2903
3121
  this.gracefulShutdownForReload()
2904
3122
  .then(() => {
@@ -3725,7 +3943,7 @@ export class FleetManager {
3725
3943
  finish(false);
3726
3944
  });
3727
3945
  }
3728
- async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch, noInlineWake = false, stillCurrent) {
3946
+ async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch, noInlineWake = false, stillCurrent, onStartRetryHold) {
3729
3947
  if (this.shuttingDown || !this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
3730
3948
  return false;
3731
3949
  await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
@@ -3764,7 +3982,7 @@ export class FleetManager {
3764
3982
  const log = idleGateTimeoutLog(instanceName, timeoutMs, this.instanceStateCache.get(instanceName), Date.now());
3765
3983
  this.logger[log.level](log.fields, log.message);
3766
3984
  }
3767
- const sent = await this.sendWhenConnected(instanceName, payload, deliveryEpoch, stillCurrent);
3985
+ const sent = await this.sendWhenConnected(instanceName, payload, deliveryEpoch, stillCurrent, onStartRetryHold);
3768
3986
  if (!sent)
3769
3987
  return false;
3770
3988
  if (this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent)) {
@@ -3793,8 +4011,11 @@ export class FleetManager {
3793
4011
  * Ordering is preserved by serialising behind any waiter already queued for this
3794
4012
  * instance, *including* when the socket happens to be up: otherwise a message
3795
4013
  * arriving after the reconnect could overtake one that has been waiting for it.
4014
+ *
4015
+ * `onStartRetryHold` (#2484, a person's message): an instance in its own start-retry backoff is waited for until that
4016
+ * retry brings it up, however long the backoff, and fails only once the retries give up.
3796
4017
  */
3797
- async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent) {
4018
+ async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent, onStartRetryHold) {
3798
4019
  if (!this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
3799
4020
  return false;
3800
4021
  const queued = this.ipcWaitTails.get(instanceName);
@@ -3808,7 +4029,7 @@ export class FleetManager {
3808
4029
  .then(() => {
3809
4030
  if (!this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
3810
4031
  return false;
3811
- return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch, stillCurrent);
4032
+ return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch, stillCurrent, onStartRetryHold);
3812
4033
  });
3813
4034
  // The chain stores a settled-either-way promise so one failed delivery cannot
3814
4035
  // wedge every later one, and so `queued` above is safe to await unguarded.
@@ -3826,9 +4047,10 @@ export class FleetManager {
3826
4047
  }
3827
4048
  }
3828
4049
  /** Poll for the instance's IPC to come back, then send. Throws if it does not. */
3829
- async sendAfterIpcReturns(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent) {
3830
- const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
4050
+ async sendAfterIpcReturns(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent, onStartRetryHold) {
4051
+ let deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
3831
4052
  let warned = false;
4053
+ let held = false;
3832
4054
  for (;;) {
3833
4055
  if (!this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
3834
4056
  return false;
@@ -3837,6 +4059,22 @@ export class FleetManager {
3837
4059
  const ipc = this.instanceIpcClients.get(instanceName);
3838
4060
  if (ipc?.connected && ipc.send(this.bindHumanReceiptPayload(instanceName, payload)))
3839
4061
  return true;
4062
+ // #2484: while the instance waits for (or runs) its own start retry, the grace runs from now — so the message is
4063
+ // delivered once a retry brings it up, and fails only after the retries stop (they gave up, or a person stopped
4064
+ // or paused it) and a last grace passes without it.
4065
+ if (onStartRetryHold && this.inStartRetryBackoff(instanceName)) {
4066
+ deadline = Math.max(deadline, Date.now() + IPC_RECONNECT_GRACE_MS);
4067
+ if (!held) {
4068
+ held = true;
4069
+ this.logger.info({ instanceName, retry: this.startupRetries.get(instanceName)?.attempt }, "Instance is waiting for its start retry — holding the message until it is up");
4070
+ try {
4071
+ onStartRetryHold();
4072
+ }
4073
+ catch (err) {
4074
+ this.logger.debug({ err, instanceName }, "Start-retry hold notice failed");
4075
+ }
4076
+ }
4077
+ }
3840
4078
  if (Date.now() >= deadline) {
3841
4079
  throw new Error(`Instance '${instanceName}' IPC is unavailable`);
3842
4080
  }
@@ -4222,6 +4460,18 @@ export class FleetManager {
4222
4460
  this.deliveryOutbox?.markHumanReceiptUncertain(id, instanceName);
4223
4461
  }
4224
4462
  }
4463
+ /**
4464
+ * #2484: a person's message waits for its instance's own start retry (DeliveryOptions.onStartRetryHold), shown as
4465
+ * queued on the message meanwhile. A ClassicBot room message (no `msg`) has no delivery-status reactions: it only waits.
4466
+ */
4467
+ startRetryHoldFor(instanceName, msg) {
4468
+ return {
4469
+ onStartRetryHold: () => {
4470
+ if (msg?.chatId && msg.messageId)
4471
+ this.reactMessageStatus(instanceName, msg.chatId, msg.messageId, "queued", msg.threadId || undefined);
4472
+ },
4473
+ };
4474
+ }
4225
4475
  /** Synchronous steer/btw handoff, using the same evidence and owner binding. */
4226
4476
  sendHumanControl(instanceName, payload, origin, originalBody) {
4227
4477
  const ipc = this.instanceIpcClients.get(instanceName);
@@ -4262,7 +4512,7 @@ export class FleetManager {
4262
4512
  return false;
4263
4513
  this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
4264
4514
  }
4265
- const sent = await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch, options.stillCurrent);
4515
+ const sent = await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch, options.stillCurrent, options.onStartRetryHold);
4266
4516
  if (!sent)
4267
4517
  return false;
4268
4518
  // A cross-instance item arriving before the daemon observes this turn as
@@ -4273,7 +4523,7 @@ export class FleetManager {
4273
4523
  return true;
4274
4524
  }
4275
4525
  const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
4276
- const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch, options.noInlineWake === true, options.stillCurrent));
4526
+ const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch, options.noInlineWake === true, options.stillCurrent, options.onStartRetryHold));
4277
4527
  this.idleGatedDeliveryTails.set(instanceName, delivery);
4278
4528
  try {
4279
4529
  return await delivery;
@@ -4729,8 +4979,8 @@ export class FleetManager {
4729
4979
  return new WakeCoordinator({
4730
4980
  mode: target => this.deliveryWorkerMode(target),
4731
4981
  available: () => !this.shuttingDown && this.deliveryOutbox?.isOpen === true,
4732
- listPending: () => this.deliveryOutbox?.listPending() ?? [],
4733
- isPaused: target => this.lifecycle.isPaused(target),
4982
+ listPending: () => this.deliveryOutbox?.listPending().filter(row => row.kind !== WORKING_RECOVERY_KIND) ?? [],
4983
+ isPaused: target => !this.sessionOomStopped(target) && this.lifecycle.isPaused(target),
4734
4984
  pauseReason: target => readPauseReason(this.getInstanceDir(target)),
4735
4985
  isRestarting: target => this.isInstanceRestarting(target),
4736
4986
  wake: async (target) => {
@@ -4742,7 +4992,7 @@ export class FleetManager {
4742
4992
  warmCap: () => this.fleetConfig?.defaults?.warm_cap ?? 0,
4743
4993
  warmOverflow: () => this.fleetConfig?.defaults?.warm_overflow ?? DEFAULT_WARM_OVERFLOW,
4744
4994
  wokeAt: target => this.lifecycle.wokeAtFor(target),
4745
- notAcceptingReason: target => this.daemons.get(target)?.notAcceptingReason?.() ?? null,
4995
+ notAcceptingReason: target => this.sessionOomStopped(target) ? this.sessionOomStopReason(target) : this.daemons.get(target)?.notAcceptingReason?.() ?? null,
4746
4996
  notifyTarget: (target, text) => { this.notifyInstanceTopic(target, text); },
4747
4997
  notifySender: (source, text) => { this.notifyInstanceTopic(source, text); },
4748
4998
  kickPump: () => this.scheduleDeliveryOutboxPump(),
@@ -4797,7 +5047,12 @@ export class FleetManager {
4797
5047
  * is exactly the old lifecycle wake; otherwise it goes through the shared
4798
5048
  * slot admission and joins any wake already in flight for the target.
4799
5049
  */
4800
- async explicitWake(name, timeoutMs = 30_000) {
5050
+ async explicitWake(name, timeoutMs = 30_000, retryStoppedSession = false) {
5051
+ if (this.sessionOomStopped(name)) {
5052
+ if (!retryStoppedSession)
5053
+ throw new SessionOomStoppedError(false, readSessionOomStop(this.getInstanceDir(name)));
5054
+ return this.restartSingleInstance(name, { explicit: true });
5055
+ }
4801
5056
  const token = await this.acquireWakeSlot(name);
4802
5057
  try {
4803
5058
  await this.lifecycle.wake(name, timeoutMs);
@@ -4818,7 +5073,7 @@ export class FleetManager {
4818
5073
  }
4819
5074
  async changeInstancePauseState(name, action) {
4820
5075
  if (action === "wake") {
4821
- await this.explicitWake(name, 30_000);
5076
+ await this.explicitWake(name, 30_000, true);
4822
5077
  this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
4823
5078
  return "awake";
4824
5079
  }
@@ -4994,6 +5249,11 @@ export class FleetManager {
4994
5249
  /** Phase 2a: the caller's transition, for a start made from inside a restart or wake. */
4995
5250
  transition, execution) {
4996
5251
  execution?.assert();
5252
+ if (this.sessionOomStopped(name)) {
5253
+ if (resumePaused)
5254
+ return this.restartSingleInstance(name, { explicit: true });
5255
+ throw new SessionOomStoppedError(false, readSessionOomStop(this.getInstanceDir(name)));
5256
+ }
4997
5257
  // CLI single-instance cold starts bypass startAll. Start diagnostics before
4998
5258
  // any lifecycle/wake work; a shutdown must not revive the stopped sampler.
4999
5259
  if (!this.shuttingDown)
@@ -5002,7 +5262,7 @@ export class FleetManager {
5002
5262
  // into a running instance — harmless, but noisy — or race this start).
5003
5263
  this.cancelStartupRetry(name);
5004
5264
  if (resumePaused && this.lifecycle.isPaused(name)) {
5005
- await this.explicitWake(name, 30_000);
5265
+ await this.explicitWake(name, 30_000, true);
5006
5266
  // A successful wake clears the persisted pause marker and produces a
5007
5267
  // fresh instance_state snapshot. Drop any stale process error left by a
5008
5268
  // pre-pause crash so /api/fleet and `agend ls` converge on running.
@@ -5022,7 +5282,7 @@ export class FleetManager {
5022
5282
  if (resumePaused) {
5023
5283
  const status = this.getInstanceStatus(name);
5024
5284
  if (status === "crashed" || status === "stopped") {
5025
- await this.restartSingleInstance(name);
5285
+ await this.restartSingleInstance(name, { explicit: true });
5026
5286
  return;
5027
5287
  }
5028
5288
  }
@@ -5197,6 +5457,8 @@ export class FleetManager {
5197
5457
  // outcome. Retrying would start an instance the operator just stopped.
5198
5458
  if (err instanceof SupersededStartError)
5199
5459
  return false;
5460
+ if (err instanceof SessionOomStoppedError)
5461
+ await this.notifySessionOomStop(name);
5200
5462
  if (this.stopOnUnsupportedCli(name, err, what))
5201
5463
  return false;
5202
5464
  this.logger.error({ err, name }, `Failed to start ${what}`);
@@ -5217,6 +5479,11 @@ export class FleetManager {
5217
5479
  scheduleStartupRetry(name, attempt) {
5218
5480
  if (this.shuttingDown)
5219
5481
  return;
5482
+ if (this.sessionOomStopped(name)) {
5483
+ this.instanceProcessStatus.set(name, "stopped");
5484
+ this.logger.warn({ name }, this.sessionOomStopReason(name));
5485
+ return;
5486
+ }
5220
5487
  if (this.startupRetries.has(name))
5221
5488
  return;
5222
5489
  const backoff = FleetManager.STARTUP_RETRY_BACKOFF_MS;
@@ -5244,6 +5511,12 @@ export class FleetManager {
5244
5511
  this.startupRetries.delete(name);
5245
5512
  this.logger.info({ name }, "Pending automatic startup retry cancelled");
5246
5513
  }
5514
+ /** #2484: the instance's own start retry is scheduled or running, and no person paused it (a stop cancels the retry). */
5515
+ inStartRetryBackoff(name) {
5516
+ if (this.shuttingDown || this.lifecycle.isPaused(name))
5517
+ return false;
5518
+ return this.startupRetries.has(name) || this.startupRetriesRunning.has(name);
5519
+ }
5247
5520
  /** Pending automatic retry, if any (status display / tests). */
5248
5521
  pendingStartupRetry(name) {
5249
5522
  const pending = this.startupRetries.get(name);
@@ -5294,6 +5567,8 @@ export class FleetManager {
5294
5567
  this.startupRetries.delete(name);
5295
5568
  if (this.shuttingDown)
5296
5569
  return;
5570
+ if (this.sessionOomStopped(name))
5571
+ return;
5297
5572
  if (!this.isConfiguredInstance(name))
5298
5573
  return; // removed from fleet.yaml / classic channel meanwhile
5299
5574
  if (this.lifecycle.isPaused(name) || this.daemons.has(name))
@@ -5309,6 +5584,7 @@ export class FleetManager {
5309
5584
  }
5310
5585
  const topicMode = this.fleetConfig?.channel?.mode === "topic"
5311
5586
  || !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
5587
+ this.startupRetriesRunning.add(name);
5312
5588
  try {
5313
5589
  await this.spawnGate.run({
5314
5590
  instanceName: name,
@@ -5330,6 +5606,9 @@ export class FleetManager {
5330
5606
  this.logger.error({ err, name, attempt: attempt + 1 }, "Automatic startup retry failed");
5331
5607
  this.scheduleStartupRetry(name, attempt + 1);
5332
5608
  }
5609
+ finally {
5610
+ this.startupRetriesRunning.delete(name);
5611
+ }
5333
5612
  }
5334
5613
  /** Fleet-topic (fleet.yaml) or ClassicBot (classic channel) — both are retryable; anything else is gone. */
5335
5614
  isConfiguredInstance(name) {
@@ -5655,12 +5934,29 @@ export class FleetManager {
5655
5934
  }
5656
5935
  /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
5657
5936
  async restartSingleInstance(name, opts) {
5937
+ if (opts?.recoveryCurrent && !opts.recoveryCurrent())
5938
+ throw new Error(t("exit.expired", name));
5939
+ const retry = opts?.explicit ? opts.sessionOomRetry ?? captureSessionOomRetry(this.getInstanceDir(name)) : undefined;
5940
+ if (opts?.sessionOomRetry) {
5941
+ const current = captureSessionOomRetry(this.getInstanceDir(name));
5942
+ // An unclear-cause stop intentionally has no tombstone. Only its live
5943
+ // launch fence can authorize an absent-marker receipt; positive stops
5944
+ // still require the offered durable revision.
5945
+ if ((!retry?.revision && !opts.recoveryCurrent) || current.revision !== retry?.revision || current.legacyRevision !== retry?.legacyRevision)
5946
+ throw new SessionOomStoppedError(true);
5947
+ }
5948
+ if (!opts?.explicit && this.sessionOomStopped(name))
5949
+ throw new SessionOomStoppedError(false, readSessionOomStop(this.getInstanceDir(name)));
5658
5950
  // One restart at a time per instance. Multiple sources can ask concurrently
5659
5951
  // (MCP revival, /restart, pty_error, model failover); a second stop/start
5660
5952
  // interleaved with the first tears down the window the first just created.
5661
5953
  // Later callers join the in-flight restart instead — its opts win.
5662
5954
  const inFlight = this.restartsInFlight.get(name);
5663
5955
  if (inFlight) {
5956
+ // Do not let a late explicit caller clear a stop belonging to the
5957
+ // automatic flight it would join. That action must be retried afterwards.
5958
+ if (retry && this.sessionOomStopped(name))
5959
+ throw new SessionOomStoppedError(true);
5664
5960
  this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
5665
5961
  // #1753: a credential-profile switch joining it still needs its conversation carried in that restart's stop →
5666
5962
  // start gap: handed to the restart while it has not reached the gap (and carries nothing else); otherwise not
@@ -5687,7 +5983,7 @@ export class FleetManager {
5687
5983
  workingDirectory,
5688
5984
  reason: "restart",
5689
5985
  stage: "lifecycle",
5690
- }, () => this.doRestartSingleInstance(name, opts, transition)))
5986
+ }, () => this.doRestartSingleInstance(name, opts, transition, retry)))
5691
5987
  .finally(() => {
5692
5988
  this.restartsInFlight.delete(name);
5693
5989
  if (this.restartGaps.get(name) === gap)
@@ -5728,7 +6024,7 @@ export class FleetManager {
5728
6024
  this.lifecycle.clearActivitySeedNow(name);
5729
6025
  }
5730
6026
  }
5731
- async doRestartSingleInstance(name, opts, transition) {
6027
+ async doRestartSingleInstance(name, opts, transition, retry) {
5732
6028
  if (this.configPath) {
5733
6029
  this.loadConfig(this.configPath);
5734
6030
  this.routing.rebuild(this.fleetConfig);
@@ -5746,7 +6042,7 @@ export class FleetManager {
5746
6042
  if (!wasPaused)
5747
6043
  this.restartsHoldingSlot.add(name);
5748
6044
  try {
5749
- await this.doRestartWithSlot(name, opts, transition, config, wasPaused);
6045
+ await this.doRestartWithSlot(name, opts, transition, config, wasPaused, retry);
5750
6046
  // An explicit restart of a paused instance is an operator wake (§1.6):
5751
6047
  // its success lifts the coordinator's park/backoff (the level stays).
5752
6048
  // Automatic restarts never do.
@@ -5758,7 +6054,7 @@ export class FleetManager {
5758
6054
  this.releaseWakeSlot(name, slot);
5759
6055
  }
5760
6056
  }
5761
- async doRestartWithSlot(name, opts, transition, config, wasPaused) {
6057
+ async doRestartWithSlot(name, opts, transition, config, wasPaused, retry) {
5762
6058
  const pausedAt = wasPaused ? (this.lifecycle.getLastPausedAt(name) ?? readPausedAt(this.getInstanceDir(name))) : null;
5763
6059
  const pauseReason = wasPaused ? readPauseReason(this.getInstanceDir(name)) : null;
5764
6060
  // #1753: the directory the CLI being stopped was launched in — its conversation is under that directory's key.
@@ -5766,6 +6062,12 @@ export class FleetManager {
5766
6062
  // Between the stop and the start: the old CLI's transcript is complete, the new one has not looked yet.
5767
6063
  /** `launching`: the login the start will use — read only when a switch is to be carried. */
5768
6064
  const betweenStopAndStart = async (launching) => {
6065
+ // Clearing is synchronous, after the old daemon is stopped and inside
6066
+ // this action's serialized transition; an unseen later stop is retained.
6067
+ if (retry)
6068
+ clearSessionOomStopForRetry(this.getInstanceDir(name), retry);
6069
+ else
6070
+ assertNoSessionOomStop(this.getInstanceDir(name));
5769
6071
  // #1753: this restart's own carry, or one a caller joined it with before this point (restartSingleInstance).
5770
6072
  const gap = this.restartGaps.get(name);
5771
6073
  const carry = gap?.carry ?? opts?.carry;
@@ -6533,7 +6835,8 @@ export class FleetManager {
6533
6835
  const [adapterId, userId] = k.split("\u0000");
6534
6836
  accounts.set(k, { adapterId, userId, sessions: 0, lastSeen: null });
6535
6837
  }
6536
- for (const key of Object.keys(cfg?.narrow ?? {})) {
6838
+ // #2348: every granted account is listed too, signed in or not.
6839
+ for (const key of [...Object.keys(cfg?.narrow ?? {}), ...Object.keys(cfg?.grants ?? {})]) {
6537
6840
  const at = key.indexOf(":"), adapterId = key.slice(0, at), userId = key.slice(at + 1), k = `${adapterId}\u0000${userId}`;
6538
6841
  if (at > 0 && !accounts.has(k))
6539
6842
  accounts.set(k, { adapterId, userId, sessions: 0, lastSeen: null });
@@ -6542,10 +6845,153 @@ export class FleetManager {
6542
6845
  const owner = { adapterId: a.adapterId, userId: a.userId };
6543
6846
  const badge = this.guestBadge(owner);
6544
6847
  // `reach`: before narrowing — what Settings offers to narrow to (#2104 review: never an agent it cannot reach).
6848
+ const grant = own(cfg?.grants, `${a.adapterId}:${a.userId}`);
6849
+ const sent = this.guestCodeSent.get(`${a.adapterId}:${a.userId}`);
6545
6850
  return { ...a, name: badge.name, platform: badge.platform, scope: [...this.guestScope(owner)].sort(), reach: [...this.guestReach(owner)].sort(),
6546
- devices: devices.get(k) ?? 0, narrow: own(cfg?.narrow, `${a.adapterId}:${a.userId}`) ?? null };
6851
+ devices: devices.get(k) ?? 0, narrow: own(cfg?.narrow, `${a.adapterId}:${a.userId}`) ?? null,
6852
+ // #2348: the grant as fleet.yaml has it (null: none), and the last code an admin sent from Settings.
6853
+ grant: grant ? { ...(grant.name ? { name: grant.name } : {}), instances: [...grant.instances] } : null,
6854
+ ...(sent ? { code: { at: sent.at, outcome: sent.outcome.kind, ...(sent.outcome.kind === "sent" ? { expiresAt: sent.at + sent.outcome.expiresInMs } : {}),
6855
+ ...(sent.outcome.kind === "dm_failed" ? { reason: sent.outcome.reason, platform: sent.outcome.platform, ...(sent.outcome.startUrl ? { startUrl: sent.outcome.startUrl } : {}) } : {}) } } : {}) };
6547
6856
  }).sort((x, y) => (y.lastSeen ?? -1) - (x.lastSeen ?? -1));
6548
- return { enabled: cfg?.enabled === true, chat: cfg?.chat !== false, instances: Object.keys(this.fleetConfig?.instances ?? {}).sort(), connections, guests };
6857
+ // #2348: `explicit` — grants are the scope (default deny); false: the derived scope of #1920, until grants are saved.
6858
+ return { enabled: cfg?.enabled === true, chat: cfg?.chat !== false, explicit: cfg?.grants !== undefined, instances: Object.keys(this.fleetConfig?.instances ?? {}).sort(), connections, guests,
6859
+ audit: this.guestAudit() };
6860
+ }
6861
+ /**
6862
+ * #2499 review: the guest audit an admin reads — the newest grant changes and admin-sent codes from the event log
6863
+ * (`fleet` rows: no agent's Log shows them). Through guestOverview only: Settings' admin gate, never over the public
6864
+ * link, never a guest's. The person's name as the card shows it; a session handle stays on the server (`by` → whether
6865
+ * a person saved it here or it was confirmed in General).
6866
+ */
6867
+ guestAudit(limit = FleetManager.GUEST_AUDIT_SHOWN) {
6868
+ if (!this.eventLog)
6869
+ return [];
6870
+ // One read, newest first by row id: a total order (created_at is to the second), so the cap never drops a newer row.
6871
+ let rows = [];
6872
+ try {
6873
+ rows = this.eventLog.latestOfTypes("fleet", ["guest_grant_changed", "guest_code_sent"], limit);
6874
+ }
6875
+ catch {
6876
+ return [];
6877
+ }
6878
+ const names = (list) => (Array.isArray(list) ? list.filter((n) => typeof n === "string") : []);
6879
+ return rows.flatMap((r) => {
6880
+ const p = r.payload ?? {}, account = typeof p.account === "string" ? p.account : "";
6881
+ const at = account.indexOf(":");
6882
+ if (at <= 0)
6883
+ return [];
6884
+ const name = this.webPerson({ adapterId: account.slice(0, at), userId: account.slice(at + 1) }).name;
6885
+ const when = Date.parse(r.created_at.includes("T") ? r.created_at : `${r.created_at.replace(" ", "T")}Z`);
6886
+ return r.event_type === "guest_grant_changed"
6887
+ ? [{ kind: "grant", at: when, account, name, via: p.via === "confirmed" ? "confirmed" : "direct", added: names(p.added), removed: names(p.removed),
6888
+ endedSessions: typeof p.endedSessions === "number" ? p.endedSessions : 0 }]
6889
+ : [{ kind: "code", at: when, account, name, outcome: typeof p.outcome === "string" ? p.outcome : "unknown" }];
6890
+ });
6891
+ }
6892
+ static GUEST_AUDIT_SHOWN = 20;
6893
+ /**
6894
+ * #2348: Settings → Guests' picker — the people AgEnD has seen in chat (seen_people, from human receipts: never a bot,
6895
+ * never the web), without fleet admins (they sign in as admins) or accounts of a connection the fleet no longer has.
6896
+ * Admin only (settings-api.ts), never over the public link.
6897
+ */
6898
+ guestPeople() {
6899
+ const seen = this.deliveryOutbox?.isOpen ? this.deliveryOutbox.seenPeople() : [];
6900
+ const people = seen.filter((p) => this.worlds.has(p.adapterId) && !this.isFleetAdmin(p.userId, p.adapterId)).map((p) => {
6901
+ const owner = { adapterId: p.adapterId, userId: p.userId };
6902
+ return { adapterId: p.adapterId, userId: p.userId, platform: this.worlds.get(p.adapterId)?.adapter.type ?? p.adapterId, username: p.username,
6903
+ name: this.webPerson(owner).name, lastSeen: p.lastSeen, instances: p.instances.map((i) => i.name) };
6904
+ });
6905
+ return { people };
6906
+ }
6907
+ /**
6908
+ * #2348: why this account may not be given a grant, or null. Only someone AgEnD has seen in chat, on a connection the
6909
+ * fleet runs, who is not a fleet admin of it (an admin is never a guest).
6910
+ */
6911
+ guestGrantRefusal(adapterId, userId) {
6912
+ if (!this.worlds.has(adapterId) && !this.fleetConfig?.channels?.some((c) => (c.id ?? c.type) === adapterId))
6913
+ return "no such connection in this fleet";
6914
+ if (this.isFleetAdmin(userId, adapterId))
6915
+ return "a fleet admin signs in as an admin, not as a guest";
6916
+ if (!this.deliveryOutbox?.isOpen || !this.deliveryOutbox.isSeenPerson(adapterId, userId))
6917
+ return "only someone AgEnD has seen in chat can be given guest access";
6918
+ return null;
6919
+ }
6920
+ /**
6921
+ * #2348 delta §5: a grant changed with a save — one event-log entry per person, shown in Settings → Guests' Recent
6922
+ * changes (guestOverview: admin-only, never over the public link or to a guest): who saved it (an admin's
6923
+ * session handle, the header token, or null when replayed for a confirmation), whether it applied at once or was
6924
+ * confirmed in General, the agents added and taken away, and the sessions a removal ended. Never message content.
6925
+ */
6926
+ noteGuestGrantChange(change) {
6927
+ this.logger.info({ account: change.account, via: change.via, added: change.added.length, removed: change.removed.length }, "Guest access changed in Settings");
6928
+ try {
6929
+ this.eventLog?.insert("fleet", "guest_grant_changed", { ...change });
6930
+ }
6931
+ catch { /* the log line stands */ }
6932
+ }
6933
+ /** #2348: an admin's Send sign-in code, per admin (the caller): at most this many in ADMIN_CODE_WINDOW_MS. */
6934
+ static ADMIN_CODE_MAX = 10;
6935
+ static ADMIN_CODE_WINDOW_MS = 10 * 60_000;
6936
+ adminCodeSends = new Map();
6937
+ /** #2348: the last admin-sent code per account, for the Settings card ("Code sent 18:42 · expires 18:52", or why not). */
6938
+ guestCodeSent = new Map();
6939
+ /**
6940
+ * #2348: Settings → Guests' "Send sign-in code" — a guest code to an account an admin granted access, by DM only (no
6941
+ * chat to answer in, nothing posted anywhere). The same issuance as the person's own /web (one code per account, the
6942
+ * code store's caps and breaker, the public link's audience when it is open) and the same once-a-minute limit per
6943
+ * account, plus a per-admin cap. Refused without guest sign-in, for a fleet admin, or with nothing granted. Audited.
6944
+ */
6945
+ async sendGuestCodeByAdmin(owner, by,
6946
+ /** #2365 review: the initiating admin is still signed in and still an admin (settings-api re-checks the session). */
6947
+ current = () => true) {
6948
+ const guests = this.fleetConfig?.web?.guests;
6949
+ if (guests?.enabled !== true)
6950
+ return { status: 409, body: { ok: false, outcome: "guests_off", error: "Guest sign-in is off" } };
6951
+ if (this.isFleetAdmin(owner.userId, owner.adapterId))
6952
+ return { status: 409, body: { ok: false, outcome: "admin", error: "A fleet admin signs in as an admin" } };
6953
+ if (!this.guestScope(owner).size)
6954
+ return { status: 409, body: { ok: false, outcome: "nothing", error: "Nothing is granted to this person yet: save their access first" } };
6955
+ const token = this.webToken;
6956
+ const adapter = this.adapters.get(owner.adapterId);
6957
+ if (this.shuttingDown || !token || !adapter)
6958
+ return { status: 503, body: { ok: false, outcome: "starting", error: "AgEnD is starting or this connection is not running" } };
6959
+ const key = `${owner.adapterId}\u0000${owner.userId}`, asked = this.guestCodeAskedAt.get(key);
6960
+ const now = performance.now();
6961
+ if (this.guestCodeInFlight.has(key) || (asked !== undefined && now - asked < 60_000)) {
6962
+ return { status: 429, body: { ok: false, outcome: "wait", retryAfterMs: asked !== undefined ? Math.max(0, 60_000 - (now - asked)) : 60_000 } };
6963
+ }
6964
+ const mine = (this.adminCodeSends.get(by) ?? []).filter((at) => now - at < FleetManager.ADMIN_CODE_WINDOW_MS);
6965
+ if (mine.length >= FleetManager.ADMIN_CODE_MAX)
6966
+ return { status: 429, body: { ok: false, outcome: "admin_limit", retryAfterMs: FleetManager.ADMIN_CODE_WINDOW_MS - (now - mine[0]) } };
6967
+ this.adminCodeSends.set(by, [...mine, now]);
6968
+ while (this.adminCodeSends.size > 256)
6969
+ this.adminCodeSends.delete(this.adminCodeSends.keys().next().value);
6970
+ let outcome = { kind: "busy" };
6971
+ this.guestCodeInFlight.add(key);
6972
+ try {
6973
+ await this.issueGuestCode({ replyHere: async () => undefined }, owner, key, adapter, token, (o) => { outcome = o; }, current);
6974
+ }
6975
+ finally {
6976
+ this.guestCodeInFlight.delete(key);
6977
+ }
6978
+ const done = outcome;
6979
+ // #2365 review: the admin's session ended (or stopped being an admin's) while the code was on its way: nothing was
6980
+ // sent, or what was made is withdrawn — the answer says so, as to anyone no longer signed in.
6981
+ if (done.kind === "revoked" && !current())
6982
+ return { status: 401, body: { ok: false, outcome: "revoked", error: "no longer authorized" } };
6983
+ this.guestCodeSent.set(`${owner.adapterId}:${owner.userId}`, { at: Date.now(), outcome: done });
6984
+ while (this.guestCodeSent.size > 512)
6985
+ this.guestCodeSent.delete(this.guestCodeSent.keys().next().value);
6986
+ // Audited: who asked, for whom, what happened (never the code).
6987
+ this.logger.info({ by, adapterId: owner.adapterId, outcome: done.kind }, "Guest sign-in code sent by an admin from Settings");
6988
+ try {
6989
+ this.eventLog?.insert("fleet", "guest_code_sent", { by, account: `${owner.adapterId}:${owner.userId}`, outcome: done.kind, instances: this.guestScope(owner).size });
6990
+ }
6991
+ catch { /* the log line stands */ }
6992
+ const status = done.kind === "sent" ? 200 : done.kind === "dm_failed" ? 502 : 409;
6993
+ return { status, body: { ok: done.kind === "sent", outcome: done.kind, ...(done.kind === "sent" ? { expiresAt: Date.now() + done.expiresInMs, publicLink: done.publicLink } : {}),
6994
+ ...(done.kind === "dm_failed" ? { platform: done.platform, reason: done.reason, ...(done.startUrl ? { startUrl: done.startUrl } : {}) } : {}) } };
6549
6995
  }
6550
6996
  /** #1920 phase 3 (the app shell): a guest's chip — its web name, and its connection's platform (discord, telegram). */
6551
6997
  guestBadge(owner) {
@@ -6614,24 +7060,35 @@ export class FleetManager {
6614
7060
  return true;
6615
7061
  }
6616
7062
  /** guestWebCommand's issuance, one per account at a time; a revoke made meanwhile (either kind) ends it unissued. */
6617
- async issueGuestCode(ask, owner, key, adapter, token) {
7063
+ async issueGuestCode(ask, owner, key, adapter, token,
7064
+ /** #2348: how it ended, for a caller with no chat to answer in (Settings → Guests' Send sign-in code). */
7065
+ report = () => { },
7066
+ /**
7067
+ * #2365 review: the one who asked may still ask — for an admin's Send sign-in code, that admin's session as it is
7068
+ * now. Asked with the revocation checks, after every wait: false ends the issuance, and withdraws its code if one was made.
7069
+ */
7070
+ stillAllowed = () => true) {
6618
7071
  const say = (text) => ask.replyHere(text).catch(() => undefined);
6619
7072
  const revokes = this.guestRevokes, accountRevokes = this.guestAccountRevokes.get(key) ?? 0;
6620
- const unrevoked = () => this.guestRevokes === revokes && (this.guestAccountRevokes.get(key) ?? 0) === accountRevokes;
7073
+ const unrevoked = () => this.guestRevokes === revokes && (this.guestAccountRevokes.get(key) ?? 0) === accountRevokes && stillAllowed();
6621
7074
  // §3.2: its rooms are read at sign-in (bounded; never longer than the reads' own deadline).
6622
7075
  await withinBudget(this.refreshGuestRooms(owner, { mode: "force" }), performance.now() + 6_000).catch(() => undefined);
6623
- if (!unrevoked())
6624
- return; // revoked while its rooms were read: no code
7076
+ if (!unrevoked()) {
7077
+ report({ kind: "revoked" });
7078
+ return;
7079
+ } // revoked while its rooms were read: no code
6625
7080
  const scope = this.guestScope(owner);
6626
7081
  if (!scope.size) {
6627
7082
  const open = this.worlds.get(owner.adapterId)?.accessManager.getMode() === "open";
6628
7083
  await say(t(open ? "web.guest_not_set_up" : "web.guest_nothing"));
7084
+ report({ kind: "nothing" });
6629
7085
  return;
6630
7086
  }
6631
7087
  this.initializeWebSessions();
6632
7088
  const codes = this.webGuestLoginCodes;
6633
7089
  if (codes.pausedForMs() > 0) {
6634
7090
  await say(t("web.guest_paused"));
7091
+ report({ kind: "paused" });
6635
7092
  return;
6636
7093
  }
6637
7094
  // §5.5 (step 4b): with the public link open, the code works through it too (an admin opened it; a guest is still a
@@ -6642,6 +7099,7 @@ export class FleetManager {
6642
7099
  recipient: { adapterId: owner.adapterId, userId: owner.userId, ...(ask.username ? { username: ask.username } : {}) } });
6643
7100
  if (!login) {
6644
7101
  await say(t("web.guest_busy"));
7102
+ report({ kind: "busy" });
6645
7103
  return;
6646
7104
  }
6647
7105
  this.guestCodeAskedAt.delete(key);
@@ -6676,13 +7134,17 @@ export class FleetManager {
6676
7134
  throw new Error("guest no longer admitted");
6677
7135
  await say(t("web.guest_sent"));
6678
7136
  }
7137
+ report({ kind: "sent", expiresInMs: LOGIN_CODE_TTL_MS, publicLink: !!open });
6679
7138
  }
6680
7139
  catch (err) {
6681
7140
  codes.revokeIfCurrent(login.issuanceId); // this flow's code only, never a later one's
6682
7141
  this.logger.info({ err: err?.message, adapterId: owner.adapterId }, "Guest sign-in code was not delivered privately");
6683
- if (!unrevoked())
6684
- return; // revoked meanwhile: the revoke's own answer said so
7142
+ if (!unrevoked()) {
7143
+ report({ kind: "revoked" });
7144
+ return;
7145
+ } // revoked meanwhile: the revoke's own answer said so
6685
7146
  const start = adapter.type === "telegram" ? this.tgStartWebUrl(owner.adapterId) : null;
7147
+ report({ kind: "dm_failed", platform: adapter.type, reason: String(err?.message ?? err).slice(0, 200), ...(start ? { startUrl: start } : {}) });
6686
7148
  // #1742 for guests (step 4b-3): its START button (a bare /start, or the link's /start web) then asks again, from
6687
7149
  // the DM — once, within the window.
6688
7150
  if (adapter.type === "telegram")
@@ -7229,6 +7691,7 @@ export class FleetManager {
7229
7691
  const startupStartedAt = Date.now();
7230
7692
  FleetManager.signalTarget = this;
7231
7693
  this.startupComplete = false;
7694
+ this.workingRecovery?.reset();
7232
7695
  this.startupDashboard = null;
7233
7696
  // Cleared here, not at the end of doStopAll: a stop has an async tail, and
7234
7697
  // anything arriving during it is still part of the stop.
@@ -7807,12 +8270,17 @@ export class FleetManager {
7807
8270
  // health server — so `systemctl start` returned success while the fleet was
7808
8271
  // still deaf: no path existed for a user message to arrive.
7809
8272
  sdNotify("READY=1");
8273
+ this.workingRecovery?.ready();
7810
8274
  // #2481: this version runs — the CLI's start-failure banner stops, and this is the version to go back to later.
7811
8275
  try {
7812
8276
  const { ownPackage, recordStarted } = await import("./start-failure.js");
7813
8277
  const version = ownPackage().version;
7814
8278
  if (version)
7815
8279
  recordStarted(this.dataDir, version, process.pid, Date.now());
8280
+ // #2481: a start completed — the failed tree an automatic restore kept for a report can go.
8281
+ const prefix = ownPackage().prefix;
8282
+ if (prefix)
8283
+ void (await import("./mac-auto-restore.js")).pruneFailedTrees(prefix).catch(() => { });
7816
8284
  }
7817
8285
  catch (err) {
7818
8286
  this.logger.debug({ err }, "fleet-started record not written");
@@ -9702,7 +10170,7 @@ export class FleetManager {
9702
10170
  ...generalReactions.meta,
9703
10171
  ...extraMeta,
9704
10172
  },
9705
- }, msg.humanOrigin, msg.humanOriginalBody ?? msg.text);
10173
+ }, msg.humanOrigin, msg.humanOriginalBody ?? msg.text, this.startRetryHoldFor(generalInstance, msg));
9706
10174
  generalReactions.consume();
9707
10175
  this.afterUserInboundDelivered(generalInstance, msg);
9708
10176
  this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
@@ -9798,7 +10266,7 @@ export class FleetManager {
9798
10266
  ...reactions.meta,
9799
10267
  ...extraMeta,
9800
10268
  },
9801
- }, msg.humanOrigin, msg.humanOriginalBody ?? msg.text);
10269
+ }, msg.humanOrigin, msg.humanOriginalBody ?? msg.text, this.startRetryHoldFor(instanceName, msg));
9802
10270
  // Only after the message actually went out. A failed delivery keeps the
9803
10271
  // reactions queued for the retry / the next message.
9804
10272
  reactions.consume();
@@ -10477,6 +10945,7 @@ export class FleetManager {
10477
10945
  // overwritten with coordinates the target's adapter cannot address.
10478
10946
  ...(foreignWorld ? {} : { chat_id: reply_chat_id, thread_id: reply_thread_id }),
10479
10947
  user: "scheduler",
10948
+ source: "schedule",
10480
10949
  ...(adapterId ? { adapter_id: adapterId } : {}),
10481
10950
  },
10482
10951
  }, { waitForIdle: true, ...(retryCurrent ? { stillCurrent: retryCurrent } : {}) });
@@ -14571,7 +15040,7 @@ export class FleetManager {
14571
15040
  ...(input.instance !== undefined ? { target: { adapterId: this.getInstanceAdapterId(input.instance) } } : {}),
14572
15041
  isAdmin: (userId, adapterId) => this.adminGate(userId, adapterId) === "ok",
14573
15042
  authorization: input.session ? this.sessionAuthorizationStore()?.get(input.session.handle) ?? null : null,
14574
- policy: { ...webAuthorityPolicy(this.fleetConfig?.web), enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink }, epoch: state.epoch, now: Date.now(),
15043
+ policy: { ...webAuthorityPolicy(this.fleetConfig?.web), enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink, quickLink: state.quickLink }, epoch: state.epoch, now: Date.now(),
14575
15044
  });
14576
15045
  }
14577
15046
  sessionAuthorizations = null;
@@ -14596,13 +15065,13 @@ export class FleetManager {
14596
15065
  session, gateway, peer, tokenEpoch: this.webToken ? tokenEpoch(this.webToken) : null,
14597
15066
  isAdmin: (userId, adapterId) => this.adminGate(userId, adapterId) === "ok",
14598
15067
  authorization: { handle: session.handle, owner: session.owner, authorizedAt: now, until: now + policy.maxSessionMs, epoch: state.epoch },
14599
- policy: { ...policy, enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink }, epoch: state.epoch, now,
15068
+ policy: { ...policy, enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink, quickLink: state.quickLink }, epoch: state.epoch, now,
14600
15069
  });
14601
15070
  if (check.refusal)
14602
15071
  return { ok: false, refusal: check.refusal };
14603
15072
  const live = new Set(this.webSessions?.list(undefined, () => true).map(r => r.handle) ?? []);
14604
15073
  store.prune(a => a.epoch === state.epoch && now <= Math.min(a.until, a.authorizedAt + policy.maxSessionMs) && live.has(a.handle));
14605
- return { ok: true, ...store.authorize(session.handle, session.owner, now, state.epoch, now + policy.maxSessionMs) };
15074
+ return { ok: true, ...store.authorize(session.handle, session.owner, now, state.epoch, now + policy.maxSessionMs, peer) };
14606
15075
  }
14607
15076
  authorityStateStore = null;
14608
15077
  /**
@@ -14611,13 +15080,24 @@ export class FleetManager {
14611
15080
  */
14612
15081
  webAuthorityState() {
14613
15082
  if (!this.dataDir)
14614
- return { enabled: false, loopbackOnly: true, namedLink: false, epoch: 0 };
15083
+ return { enabled: false, loopbackOnly: true, namedLink: false, quickLink: false, epoch: 0 };
14615
15084
  this.authorityStateStore ??= new AuthorityStateStore(join(this.dataDir, "web-authority.json"), this.logger, undefined, undefined, () => { this.ensureDeliveryOutbox?.(); return this.deliveryOutbox?.maxWebAuthorityEpoch() ?? 0; });
14616
15085
  const state = this.authorityStateStore.current(this.fleetConfig?.web?.verified_authority);
14617
- // #2453 (§0.2): the named-link option seen going off — the authority of every session over the link ends now, for
14618
- // good: their authorizations go (a later "on" can never bring one back without its account's Confirm).
14619
- if (state.namedLinkClosed)
14620
- this.sessionAuthorizationStore()?.prune(a => this.webSessions?.findByHandle(a.handle)?.surface === "local");
15086
+ // #2453 (§0.2) / §0.3: a link's option seen going off — the authority of every session over that link ends now, for
15087
+ // good: their authorizations go (a later "on" can never bring one back without its account's Confirm). A record from
15088
+ // before §0.3 has no peer: over the gateway, only a named-link session could have been authorized then.
15089
+ if (state.namedLinkClosed || state.quickLinkClosed) {
15090
+ const ended = (a) => {
15091
+ const session = this.webSessions?.findByHandle(a.handle);
15092
+ if (!session)
15093
+ return true;
15094
+ if (session.surface === "local")
15095
+ return false;
15096
+ const peer = a.peer ?? "named_link";
15097
+ return (!!state.namedLinkClosed && peer === "named_link") || (!!state.quickLinkClosed && peer === "quick_link");
15098
+ };
15099
+ this.sessionAuthorizationStore()?.prune(a => !ended(a));
15100
+ }
14621
15101
  return state;
14622
15102
  }
14623
15103
  /**
@@ -14650,7 +15130,8 @@ export class FleetManager {
14650
15130
  .filter(a => a.epoch === state.epoch && sessions.has(a.handle) && now <= Math.min(a.until, a.authorizedAt + maxMs))
14651
15131
  .map(a => ({ handle: a.handle, label: sessions.get(a.handle).label, account: `${a.owner.adapterId}:${a.owner.userId}`, authorizedAt: a.authorizedAt,
14652
15132
  until: Math.min(a.until, a.authorizedAt + maxMs, sessions.get(a.handle).absoluteExpiry),
14653
- ...(sessions.get(a.handle).surface === "gateway" ? { named: true } : {}) })) : [];
15133
+ // Over a link: the temporary one (§0.3), else the named one (a record from before §0.3 has no peer).
15134
+ ...(sessions.get(a.handle).surface === "gateway" ? (a.peer === "quick_link" ? { quick: true } : { named: true }) : {}) })) : [];
14654
15135
  const host = this.fleetConfig?.web?.public_link?.hostname;
14655
15136
  return { state, configured, hours: VERIFIED_SESSION_HOURS, lanHosts, authorized, namedHost: typeof host === "string" && host ? host : null };
14656
15137
  }
@@ -14665,9 +15146,17 @@ export class FleetManager {
14665
15146
  const adapterOf = (adapterId) => this.adapters.get(adapterId) ?? this.worlds.get(adapterId)?.adapter;
14666
15147
  // The host's clock, with its zone: the person may read the DM on a phone in another one.
14667
15148
  const when = (ms) => new Date(ms).toLocaleString(undefined, { timeZoneName: "short" });
15149
+ const quickHost = () => { try {
15150
+ const u = this.publicWebLink?.openSignin()?.url;
15151
+ return u ? new URL(u).host : "?";
15152
+ }
15153
+ catch {
15154
+ return "?";
15155
+ } };
14668
15156
  const where = (peer) => peer === "named_link"
14669
15157
  ? t("web_auth.peer_named_link", String(this.fleetConfig?.web?.public_link?.hostname ?? "?")) // §0.2: the internet
14670
- : t(peer === "loopback" ? "web_auth.peer_loopback" : "web_auth.peer_lan");
15158
+ : peer === "quick_link" ? t("web_auth.peer_quick_link", quickHost()) // §0.3: the internet
15159
+ : t(peer === "loopback" ? "web_auth.peer_loopback" : "web_auth.peer_lan");
14671
15160
  return this.authorizationRequests = new SessionAuthorizationRequests({
14672
15161
  now: () => performance.now(), wallNow: () => Date.now(),
14673
15162
  maxSessionMs: () => webAuthorityPolicy(this.fleetConfig?.web).maxSessionMs,
@@ -14701,8 +15190,8 @@ export class FleetManager {
14701
15190
  const record = this.webSessions?.findByHandle(handle);
14702
15191
  if (!record)
14703
15192
  return { ok: false, refusal: "session_gone" };
14704
- // §0.2: a named-link session is judged as the gateway request it is (its exposure still the current one).
14705
- const r = this.authorizeWebSession(record, peer, peer === "named_link");
15193
+ // §0.2 / §0.3: a session over a link is judged as the gateway request it is (its exposure still the current one).
15194
+ const r = this.authorizeWebSession(record, peer, peer === "named_link" || peer === "quick_link");
14706
15195
  // What the card says: the end of the authorization, or the session's own end when that comes first.
14707
15196
  return r.ok ? { ok: true, until: Math.min(r.authorization.until, record.absoluteExpiry) } : { ok: false, refusal: r.refusal };
14708
15197
  },
@@ -14748,7 +15237,7 @@ export class FleetManager {
14748
15237
  session: s, gateway: input.gateway, peer: input.peer, tokenEpoch: this.webToken ? tokenEpoch(this.webToken) : null,
14749
15238
  isAdmin: (userId, adapterId) => this.adminGate(userId, adapterId) === "ok",
14750
15239
  authorization: s?.owner ? { handle: s.handle, owner: s.owner, authorizedAt: now, until: now + policy.maxSessionMs, epoch: state.epoch } : null,
14751
- policy: { ...policy, enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink }, epoch: state.epoch, now,
15240
+ policy: { ...policy, enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink, quickLink: state.quickLink }, epoch: state.epoch, now,
14752
15241
  });
14753
15242
  return r.refusal;
14754
15243
  }
@@ -15021,14 +15510,42 @@ export class FleetManager {
15021
15510
  /** A clean exit is intentional from the CLI's perspective, but often not from
15022
15511
  * the operator's. Keep the instance notice passive and put the action in the
15023
15512
  * same-world General topic where an administrator can make the choice. */
15024
- async notifyNormalExit(instanceName) {
15025
- this.notifyInstanceTopic(instanceName, t("exit.instance_notice", instanceName) + placeSuffix({ kind: "general" }, "General"));
15513
+ async notifySessionOomStop(instanceName, unclear = false) {
15514
+ await this.notifyNormalExit(instanceName, unclear ? "unclear" : true);
15515
+ }
15516
+ async notifyNormalExit(instanceName, oom = false) {
15517
+ const daemon = oom === "unclear" ? this.daemons.get(instanceName) : undefined;
15518
+ const owner = daemon ? { ...daemon.getInteractionSnapshot().owner } : undefined;
15519
+ const current = daemon && owner ? () => this.daemons.get(instanceName) === daemon
15520
+ && daemon.isUnclearOomStopped() && sameInteractionOwner(owner, daemon.getInteractionSnapshot().owner) : undefined;
15521
+ if (oom === "unclear" && !current?.())
15522
+ return;
15523
+ const state = oom ? readSessionOomStop(this.getInstanceDir(instanceName)) : null;
15524
+ if (oom === "unclear" && state?.blocked)
15525
+ return;
15526
+ const proof = state?.blocked ? state.record?.evidence ?? state.evidence : undefined;
15527
+ if (oom === true && !proof)
15528
+ return;
15529
+ const sessionOomRetry = oom ? captureSessionOomRetry(this.getInstanceDir(instanceName)) : undefined;
15530
+ if (oom && [...this.pendingNonceButtons.values()].some(p => p.prefix === OOM_RETRY_CALLBACK_PREFIX
15531
+ && p.instanceName === instanceName && p.sessionOomRetry?.revision === sessionOomRetry?.revision
15532
+ && (oom !== "unclear" || p.confirmationCurrent?.() === true)))
15533
+ return;
15534
+ const prefix = oom ? OOM_RETRY_CALLBACK_PREFIX : EXIT_RESTART_CALLBACK_PREFIX;
15535
+ const message = oom === "unclear" ? t("inst.oom_unclear_choice", instanceName) : oom && proof
15536
+ ? t("inst.session_oom_choice", instanceName, describeSelfOom(proof))
15537
+ : t("exit.general_notice", placeRef({ kind: "instance", name: instanceName }, instanceName));
15538
+ const choices = oom
15539
+ ? [{ action: "resume", label: t("inst.session_oom_resume") }, { action: "fresh", label: t("inst.session_oom_fresh") }]
15540
+ : [{ action: "restart", label: t("exit.restart") }, { action: "ignore", label: t("exit.ignore") }];
15541
+ const extra = sessionOomRetry ? { sessionOomRetry, ...(current ? { confirmationCurrent: current } : {}) } : undefined;
15542
+ this.notifyInstanceTopic(instanceName, (oom ? message : t("exit.instance_notice", instanceName)) + placeSuffix({ kind: "general" }, "General"));
15026
15543
  const web = this.webOnlyPromptPlace();
15027
15544
  if (web) {
15028
15545
  await this.postNonceButtonPrompt({
15029
- prefix: EXIT_RESTART_CALLBACK_PREFIX, alertType: "exit_restart", instanceName, ...web,
15030
- message: t("exit.general_notice", placeRef({ kind: "instance", name: instanceName }, instanceName)),
15031
- choices: [{ action: "restart", label: t("exit.restart") }, { action: "ignore", label: t("exit.ignore") }],
15546
+ prefix, alertType: "exit_restart", instanceName, ...web,
15547
+ message, extra,
15548
+ choices,
15032
15549
  expiredText: t("exit.expired", instanceName),
15033
15550
  });
15034
15551
  return;
@@ -15049,37 +15566,36 @@ export class FleetManager {
15049
15566
  return;
15050
15567
  }
15051
15568
  await this.postNonceButtonPrompt({
15052
- prefix: EXIT_RESTART_CALLBACK_PREFIX,
15569
+ prefix,
15053
15570
  alertType: "exit_restart",
15054
15571
  instanceName,
15055
15572
  adapter,
15056
15573
  adapterId,
15057
15574
  chatId,
15058
15575
  threadId,
15059
- message: t("exit.general_notice", placeRef({ kind: "instance", name: instanceName }, instanceName)),
15060
- choices: [
15061
- { action: "restart", label: t("exit.restart") },
15062
- { action: "ignore", label: t("exit.ignore") },
15063
- ],
15576
+ message, extra,
15577
+ choices,
15064
15578
  expiredText: t("exit.expired", instanceName),
15065
15579
  });
15066
15580
  }
15067
15581
  /** Consume a clean-exit Restart/Ignore button exactly once. */
15068
15582
  async handleExitRestartPrompt(data, callbackAdapterId, receivingAdapter) {
15069
- const claimed = this.consumeNonceCallback(EXIT_RESTART_CALLBACK_PREFIX, /^exit-restart:([0-9a-f]+):(restart|ignore)$/, data, callbackAdapterId, receivingAdapter);
15583
+ const oom = data.callbackData.startsWith(OOM_RETRY_CALLBACK_PREFIX);
15584
+ const claimed = this.consumeNonceCallback(oom ? OOM_RETRY_CALLBACK_PREFIX : EXIT_RESTART_CALLBACK_PREFIX, oom ? /^oom-retry:([0-9a-f]+):(resume|fresh)$/ : /^exit-restart:([0-9a-f]+):(restart|ignore)$/, data, callbackAdapterId, receivingAdapter);
15070
15585
  if (claimed === null)
15071
15586
  return false;
15072
15587
  if (claimed === "consumed")
15073
15588
  return true;
15074
15589
  const { entry: pending, action } = claimed;
15075
- this.eventLog?.insert(pending.instanceName, "normal_exit_action", { action, userId: data.userId });
15590
+ this.eventLog?.insert(pending.instanceName, oom ? "session_oom_action" : "normal_exit_action", { action, userId: data.userId });
15076
15591
  if (action === "ignore") {
15077
15592
  await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("exit.ignored", pending.instanceName));
15078
15593
  return true;
15079
15594
  }
15080
15595
  await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("exit.restarting", pending.instanceName));
15081
15596
  try {
15082
- await this.restartSingleInstance(pending.instanceName);
15597
+ await this.restartSingleInstance(pending.instanceName, { explicit: true, ...(oom ? { freshStart: action === "fresh", sessionOomRetry: pending.sessionOomRetry,
15598
+ ...(pending.confirmationCurrent ? { recoveryCurrent: pending.confirmationCurrent } : {}) } : {}) });
15083
15599
  await pending.adapter.editMessage(pending.chatId, pending.messageId ?? data.messageId, t("exit.restarted", pending.instanceName), pending.threadId);
15084
15600
  }
15085
15601
  catch (err) {
@@ -16910,6 +17426,10 @@ export class FleetManager {
16910
17426
  log: (level, message, extra) => this.logger[level](extra ?? {}, message),
16911
17427
  });
16912
17428
  db.board.onChange(({ rev, ids, projects, wakes }) => {
17429
+ // Refresh both old and new owners on reassignment; wakes are not an ownership index.
17430
+ if (ids.length)
17431
+ for (const name of this.daemons.keys())
17432
+ this.observeWorkingState(name, "tasks");
16913
17433
  this.emitSseEvent("tasks", { rev, ids, projects });
16914
17434
  waker.note(wakes);
16915
17435
  // step 3: cards waiting for the person are Needs you items; #2355 review: a project's lead routes them too.
@@ -17008,7 +17528,7 @@ export class FleetManager {
17008
17528
  }
17009
17529
  await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("hang.restarting", pending.instanceName));
17010
17530
  try {
17011
- await this.restartSingleInstance(pending.instanceName);
17531
+ await this.restartSingleInstance(pending.instanceName, { explicit: true });
17012
17532
  await pending.adapter.editMessage(pending.chatId, pending.messageId ?? data.messageId, t("hang.restarted", pending.instanceName), pending.threadId);
17013
17533
  }
17014
17534
  catch (err) {
@@ -17544,7 +18064,12 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
17544
18064
  const isPrimaryAdapter = !mentionWorld || mentionWorld.adapter === this.adapter;
17545
18065
  const adapterBotUserId = mentionWorld?.botUserId ?? (isPrimaryAdapter ? this.botUserId : undefined);
17546
18066
  const mentionTag = adapterBotUserId ? `<@${adapterBotUserId}>` : null;
17547
- const isMentioned = mentionTag && text.includes(mentionTag);
18067
+ // #2551: a native reply to one of THIS bot's messages is addressed to it, as a mention is (a Discord reply carries
18068
+ // its ping in the reference, not the text). One hop only: a reply to a message of ours that itself answered a bot
18069
+ // (or the sender) is a chain of bots answering each other — a mention still carries it on. A reply to our answer
18070
+ // to a person is the ordinary handoff, and is addressed.
18071
+ const repliesToUs = !!adapterBotUserId && msg.replyToAuthorId === adapterBotUserId && msg.replyToChain !== true;
18072
+ const isMentioned = (mentionTag && text.includes(mentionTag)) || repliesToUs;
17548
18073
  if (!isMentioned) {
17549
18074
  // Bare attachment (no @mention) — already saved above; just acknowledge.
17550
18075
  if (saved) {
@@ -17780,7 +18305,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
17780
18305
  content: fullText,
17781
18306
  targetSession: instanceName,
17782
18307
  meta,
17783
- }, msg.humanOrigin, msg.humanOriginalBody ?? text);
18308
+ }, msg.humanOrigin, msg.humanOriginalBody ?? text, this.startRetryHoldFor(instanceName));
17784
18309
  reactions.consume();
17785
18310
  }
17786
18311
  catch (err) {
@@ -19738,7 +20263,12 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
19738
20263
  // also sees the channel's messages.
19739
20264
  if (adapterId)
19740
20265
  this.bindInstanceAdapter(instanceName, adapterId);
19741
- await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
20266
+ if (this.sessionOomStopped(instanceName)) {
20267
+ await this.restartSingleInstance(instanceName, { explicit: true });
20268
+ }
20269
+ else {
20270
+ await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
20271
+ }
19742
20272
  this.reregisterClassicChannels();
19743
20273
  // Auto-enable collab for Discord classic channels (TG uses @mention directly without collab mode)
19744
20274
  if (guildId && !classicChannels.isCollab(channelId, adapterId)) {
@@ -19808,6 +20338,22 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
19808
20338
  }
19809
20339
  tmuxExecHelperTimer = null;
19810
20340
  stopAllInFlight = null;
20341
+ /** Signal-handler continuation, after successful graceful shutdown only. */
20342
+ noteRequestedShutdown() {
20343
+ const progress = readUpdateProgress(this.dataDir)?.progress;
20344
+ const update = progress && updateProgressOperation(progress) === "update"
20345
+ && ["installed", "stopping", "starting"].includes(progress.stage);
20346
+ this.recordWorkingRestartIntent(update ? "update" : "manual_restart");
20347
+ }
20348
+ recordWorkingRestartIntent(cause) {
20349
+ this.workingRecoveryCause = cause;
20350
+ try {
20351
+ this.deliveryOutbox?.workingRecovery.requested(this.managerBootId, cause);
20352
+ }
20353
+ catch {
20354
+ this.logger.warn("Could not record requested restart; next recovery will report an unknown interruption");
20355
+ }
20356
+ }
19811
20357
  /** Stops release memory. Reassess between batches without waiting for recovery. */
19812
20358
  async stopInstancesInBatches(entries, stop) {
19813
20359
  let index = 0;
@@ -19825,15 +20371,25 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
19825
20371
  }
19826
20372
  }
19827
20373
  async doStopAll() {
20374
+ // The scheduler closes before individual daemons stop. Preserve the owned
20375
+ // task/reply boundary while those stores are still available, not a later
20376
+ // shutdown snapshot whose unavailable sources cannot authorize recovery.
20377
+ for (const name of this.daemons.keys())
20378
+ this.observeWorkingState(name, "before_stop");
19828
20379
  this.startupMemoryDeadline?.stop();
19829
20380
  this.startupMemoryDeadline = null;
19830
20381
  this.startupComplete = false;
20382
+ this.workingRecovery?.reset();
19831
20383
  this.startupDashboard = null;
19832
20384
  this.reloadPending = false;
19833
20385
  // Before anything is stopped: everything that dies from here on dies
19834
20386
  // because we asked it to. Set synchronously — doStopAll runs to its first
19835
20387
  // await in the same tick as the signal handler, so no event can slip in.
19836
20388
  this.shuttingDown = true;
20389
+ this.workingRecovery?.reset();
20390
+ if (this.workingRecoveryTimer)
20391
+ clearInterval(this.workingRecoveryTimer);
20392
+ this.workingRecoveryTimer = null;
19837
20393
  if (this.updateProgressTimer) {
19838
20394
  clearInterval(this.updateProgressTimer);
19839
20395
  this.updateProgressTimer = null;
@@ -20513,6 +21069,27 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
20513
21069
  fleetSignatureMismatchKeys() {
20514
21070
  return this.fleetSignatureMismatch;
20515
21071
  }
21072
+ /**
21073
+ * #2347 restart flow: saved changes that take effect only after AgEnD restarts — the startup-only keys fleet.yaml
21074
+ * now differs on, and the newest apply job whose fleet row can carry the restart (`job_id`, null when none can: a
21075
+ * hand edit, or a job past its retention). Null when nothing is pending, and when a restart could not clear it
21076
+ * (the signature mismatch the panel reports instead). The same test requestSettingsSelfRestart makes.
21077
+ */
21078
+ settingsRestartPending() {
21079
+ if (this.appliedFleetLevel === null || this.fleetSignatureMismatch)
21080
+ return null;
21081
+ const next = this.nextFleetConfig();
21082
+ if (this.appliedFleetLevel === this.fleetLevelSignature(next))
21083
+ return null;
21084
+ let job = null;
21085
+ // A job whose restart was asked for already has its row in flight ("running"), so it is never offered again.
21086
+ for (const candidate of this.applyJobs.all()) {
21087
+ const row = candidate.targets.find(item => item.target === APPLY_FLEET_TARGET);
21088
+ if (row?.status === "restart-required" && (!job || candidate.startedAt >= job.startedAt))
21089
+ job = candidate;
21090
+ }
21091
+ return { keys: fleetLevelDifferences(this.startupFleetConfig, next), job_id: job ? job.id : null };
21092
+ }
20516
21093
  /**
20517
21094
  * Restart AgEnD itself on behalf of a Settings apply.
20518
21095
  *
@@ -21763,6 +22340,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
21763
22340
  this.logger.info("No instances to restart");
21764
22341
  return;
21765
22342
  }
22343
+ this.recordWorkingRestartIntent("manual_restart");
22344
+ this.workingRecovery?.reset();
21766
22345
  this.logger.info(`Graceful restart: waiting for ${instanceNames.length} instances to idle...`);
21767
22346
  const groupId = this.fleetConfig?.channel?.group_id;
21768
22347
  const generalName = this.findGeneralInstance();
@@ -21863,7 +22442,16 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
21863
22442
  this.startStatuslineWatcher(name);
21864
22443
  }
21865
22444
  }
22445
+ this.workingRecovery?.ready();
21866
22446
  this.logger.info("Graceful restart complete");
22447
+ // The in-process interval keeps its label, but a later unrelated process
22448
+ // crash must not inherit this completed request as its shutdown intent.
22449
+ try {
22450
+ this.deliveryOutbox?.workingRecovery.clearRequested(this.managerBootId);
22451
+ }
22452
+ catch {
22453
+ this.logger.warn("Could not clear completed in-process restart intent");
22454
+ }
21867
22455
  const configuredNames = this.configuredStartupInstanceNames(fleet, topicMode);
21868
22456
  const total = configuredNames.length;
21869
22457
  const started = configuredNames.filter(name => this.daemons.has(name)).length;
@@ -21918,6 +22506,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
21918
22506
  }
21919
22507
  const ipc = this.instanceIpcClients.get(name);
21920
22508
  if (ipc?.connected) {
22509
+ daemon?.noteLegacyRestartReminder?.();
21921
22510
  ipc.send({
21922
22511
  type: "fleet_inbound",
21923
22512
  content: msg,
@@ -22065,6 +22654,11 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
22065
22654
  const guests = this.fleetConfig?.web?.guests;
22066
22655
  if (!owner || guests?.enabled !== true)
22067
22656
  return NO_GUEST_SCOPE;
22657
+ // #2348: explicit grants, once the key exists, are the whole scope — no narrowing on top (a grant is already exact).
22658
+ if (guests.grants !== undefined) {
22659
+ const granted = this.guestReach(owner);
22660
+ return granted.size ? granted : NO_GUEST_SCOPE;
22661
+ }
22068
22662
  const scope = new Set(this.guestReach(owner));
22069
22663
  const narrow = guests.narrow && Object.hasOwn(guests.narrow, `${owner.adapterId}:${owner.userId}`) ? guests.narrow[`${owner.adapterId}:${owner.userId}`] : undefined;
22070
22664
  if (narrow) {
@@ -22085,6 +22679,20 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
22085
22679
  const scope = new Set();
22086
22680
  if (guests?.enabled !== true)
22087
22681
  return scope;
22682
+ // #2348: with explicit grants, what an admin granted this account — of the instances that exist now (fleet.yaml's, or
22683
+ // a ClassicBot room's) — and nothing else: no connection access, no room membership, no open_instances.
22684
+ if (guests.grants !== undefined) {
22685
+ const key = `${owner.adapterId}:${owner.userId}`;
22686
+ const grant = Object.hasOwn(guests.grants, key) ? guests.grants[key] : undefined;
22687
+ if (!grant || !Array.isArray(grant.instances))
22688
+ return scope;
22689
+ const rooms = new Set((this.classicChannels?.getAll() ?? []).map((r) => r.instanceName));
22690
+ for (const name of grant.instances) {
22691
+ if ((this.fleetConfig?.instances && Object.hasOwn(this.fleetConfig.instances, name)) || rooms.has(name))
22692
+ scope.add(name);
22693
+ }
22694
+ return scope;
22695
+ }
22088
22696
  const own = (map, key) => (map && Object.hasOwn(map, key) ? map[key] : undefined);
22089
22697
  const world = this.worlds.get(owner.adapterId);
22090
22698
  if (world && world.accessManager.isAllowed(owner.userId)) {
@@ -22126,7 +22734,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
22126
22734
  const owner = record.owner;
22127
22735
  if (this.guestScope(owner).size)
22128
22736
  return "ok";
22129
- if (owner && this.fleetConfig?.web?.guests?.enabled === true) {
22737
+ // #2348: with explicit grants no room's membership can give anything: nothing granted is gone, not pending. Nor does
22738
+ // a connection's start matter: grants are config (and ClassicBot rooms), read without the adapter.
22739
+ if (owner && this.fleetConfig?.web?.guests?.enabled === true && this.fleetConfig.web.guests.grants === undefined) {
22130
22740
  // Before adapters initialize, topic scope is unknown, not revoked. Keep the returning cookie but grant no
22131
22741
  // request until its configured world exists. Removed connections and initialized denied access stay final.
22132
22742
  const channels = this.fleetConfig.channels ?? (this.fleetConfig.channel ? [this.fleetConfig.channel] : []);