@songsid/agend 2.3.0 → 2.3.2-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-TW.md +1 -1
- package/dist/authority/authority-state.d.ts +4 -0
- package/dist/authority/authority-state.js +16 -10
- package/dist/authority/authority-state.js.map +1 -1
- package/dist/authority/session-authorizations.d.ts +7 -1
- package/dist/authority/session-authorizations.js +6 -4
- package/dist/authority/session-authorizations.js.map +1 -1
- package/dist/authority/web-authority.d.ts +19 -6
- package/dist/authority/web-authority.js +22 -8
- package/dist/authority/web-authority.js.map +1 -1
- package/dist/backend/muse.d.ts +20 -2
- package/dist/backend/muse.js +154 -16
- package/dist/backend/muse.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +2 -0
- package/dist/channel/adapters/discord.js +33 -3
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +1 -1
- package/dist/channel/adapters/telegram.js +3 -35
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/mcp-tools.js +5 -5
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/types.d.ts +11 -0
- package/dist/cli.js +6 -3
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +42 -5
- package/dist/config-validator.js.map +1 -1
- package/dist/daemon.d.ts +47 -3
- package/dist/daemon.js +380 -60
- package/dist/daemon.js.map +1 -1
- package/dist/delivery-outbox.d.ts +23 -1
- package/dist/delivery-outbox.js +114 -17
- package/dist/delivery-outbox.js.map +1 -1
- package/dist/doctor.d.ts +3 -0
- package/dist/doctor.js +9 -0
- package/dist/doctor.js.map +1 -1
- package/dist/event-log.d.ts +5 -0
- package/dist/event-log.js +13 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-context.d.ts +1 -1
- package/dist/fleet-lock.d.ts +6 -0
- package/dist/fleet-lock.js +19 -0
- package/dist/fleet-lock.js.map +1 -1
- package/dist/fleet-manager.d.ts +174 -2
- package/dist/fleet-manager.js +678 -68
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-process-handlers.d.ts +1 -0
- package/dist/fleet-process-handlers.js +9 -1
- package/dist/fleet-process-handlers.js.map +1 -1
- package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +11 -0
- package/dist/general-knowledge/skills/web-appearance/SKILL.canvas-worker-off.md +4 -4
- package/dist/general-knowledge/skills/web-appearance/SKILL.md +7 -7
- package/dist/general-knowledge/skills/web-dashboard/SKILL.canvas-worker-off.md +109 -57
- package/dist/general-knowledge/skills/web-dashboard/SKILL.md +110 -58
- package/dist/guest-seen-people.d.ts +38 -0
- package/dist/guest-seen-people.js +78 -0
- package/dist/guest-seen-people.js.map +1 -0
- package/dist/instance-config-impact.js +4 -1
- package/dist/instance-config-impact.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +2 -0
- package/dist/instance-lifecycle.js +27 -1
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +62 -12
- package/dist/locale.js.map +1 -1
- package/dist/log-service.js +1 -0
- package/dist/log-service.js.map +1 -1
- package/dist/mac-auto-restore.d.ts +117 -0
- package/dist/mac-auto-restore.js +294 -0
- package/dist/mac-auto-restore.js.map +1 -0
- package/dist/needs-you.js +1 -0
- package/dist/needs-you.js.map +1 -1
- package/dist/outbound-handlers.d.ts +2 -2
- package/dist/outbound-handlers.js +5 -5
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +2 -0
- package/dist/outbound-schemas.js +2 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/public-web-gateway.js +9 -1
- package/dist/public-web-gateway.js.map +1 -1
- package/dist/search/search-index.d.ts +10 -2
- package/dist/search/search-index.js +17 -4
- package/dist/search/search-index.js.map +1 -1
- package/dist/service-installer.d.ts +23 -1
- package/dist/service-installer.js +40 -11
- package/dist/service-installer.js.map +1 -1
- package/dist/session-oom-stop.d.ts +45 -0
- package/dist/session-oom-stop.js +259 -0
- package/dist/session-oom-stop.js.map +1 -0
- package/dist/session-oom.d.ts +13 -0
- package/dist/session-oom.js +119 -7
- package/dist/session-oom.js.map +1 -1
- package/dist/settings-api.d.ts +27 -1
- package/dist/settings-api.js +108 -2
- package/dist/settings-api.js.map +1 -1
- package/dist/settings-change.js +37 -3
- package/dist/settings-change.js.map +1 -1
- package/dist/settings-effect.js +5 -2
- package/dist/settings-effect.js.map +1 -1
- package/dist/spawn-gate.d.ts +2 -0
- package/dist/spawn-gate.js +17 -4
- package/dist/spawn-gate.js.map +1 -1
- package/dist/start-attempt.d.ts +15 -1
- package/dist/start-attempt.js +35 -5
- package/dist/start-attempt.js.map +1 -1
- package/dist/start-failure-chat.d.ts +41 -0
- package/dist/start-failure-chat.js +159 -0
- package/dist/start-failure-chat.js.map +1 -0
- package/dist/start-failure-notice.d.ts +4 -0
- package/dist/start-failure-notice.js +8 -0
- package/dist/start-failure-notice.js.map +1 -1
- package/dist/start-failure.d.ts +49 -0
- package/dist/start-failure.js +141 -1
- package/dist/start-failure.js.map +1 -1
- package/dist/telegram-api-root.d.ts +1 -0
- package/dist/telegram-api-root.js +40 -0
- package/dist/telegram-api-root.js.map +1 -0
- package/dist/turn-reply-guard.d.ts +25 -0
- package/dist/turn-reply-guard.js +111 -12
- package/dist/turn-reply-guard.js.map +1 -1
- package/dist/types.d.ts +21 -3
- package/dist/ui/backend-signin.js +5 -2
- package/dist/ui/chat-ambient.js +1 -1
- package/dist/ui/chat-effects.js +4 -4
- package/dist/ui/chat-render.js +5 -2
- package/dist/ui/chat-strings.js +2 -2
- package/dist/ui/chat-thread.js +2 -2
- package/dist/ui/first-run.js +2 -2
- package/dist/ui/fleet-canvas-view.js +114 -0
- package/dist/ui/fleet-canvas.js +15 -77
- package/dist/ui/fleet-office-editor.js +80 -5
- package/dist/ui/fleet-office-layout.js +197 -0
- package/dist/ui/fleet-office-model.js +6 -3
- package/dist/ui/fleet-office-pack.d.ts +39 -0
- package/dist/ui/fleet-office-pack.js +240 -29
- package/dist/ui/fleet-office-skin.js +130 -21
- package/dist/ui/fleet-office-store.js +75 -4
- package/dist/ui/fleet-office-view.js +154 -0
- package/dist/ui/fleet-office-zones.js +95 -33
- package/dist/ui/fleet-office.js +530 -94
- package/dist/ui/fleet-usage.js +7 -6
- package/dist/ui/panel-chat.js +20 -7
- package/dist/ui/panel-details.js +2 -2
- package/dist/ui/panel-fleet.js +11 -77
- package/dist/ui/panel-log.js +5 -5
- package/dist/ui/panel-settings.js +586 -164
- package/dist/ui/search-dialog.js +26 -7
- package/dist/ui/settings-appearance.js +82 -29
- package/dist/ui/settings-apply.js +42 -7
- package/dist/ui/settings-dialogs.js +68 -22
- package/dist/ui/settings-draft.js +51 -0
- package/dist/ui/settings-guests.js +244 -78
- package/dist/ui/settings-learn.js +2 -2
- package/dist/ui/settings-model.js +2 -0
- package/dist/ui/settings-registry.js +142 -0
- package/dist/ui/settings-strings.js +108 -44
- package/dist/ui/settings-web-authority.js +34 -23
- package/dist/ui/shared/app-i18n.js +14 -0
- package/dist/ui/shared/app-nav.js +10 -5
- package/dist/ui/shared/app-prefs.js +80 -2
- package/dist/ui/shared/app-route.js +39 -5
- package/dist/ui/shared/app-shell.js +80 -8
- package/dist/ui/shared/app-stream.js +2 -1
- package/dist/ui/shared/app.css +251 -20
- package/dist/ui/shared/app.js +3 -0
- package/dist/ui/shared/effect-stock-trust.js +1 -1
- package/dist/ui/shared/identity-color.js +4 -4
- package/dist/ui/shared/signin.js +2 -2
- package/dist/ui/shared/ui-dialog.js +6 -1
- package/dist/ui/shared/ui-icons.js +2 -0
- package/dist/ui/shared/usage-dialog.js +21 -11
- package/dist/ui/shared/view-strings.js +2 -0
- package/dist/ui/shared/web-authorize.js +13 -12
- package/dist/web-api.d.ts +9 -3
- package/dist/web-api.js +161 -6
- package/dist/web-api.js.map +1 -1
- package/dist/web-auth.js +7 -1
- package/dist/web-auth.js.map +1 -1
- package/dist/web-experimental.d.ts +1 -1
- package/dist/web-experimental.js +1 -1
- package/dist/web-experimental.js.map +1 -1
- package/dist/web-mod-agent.js +2 -2
- package/dist/web-mod-agent.js.map +1 -1
- package/dist/web-office-skins.d.ts +108 -0
- package/dist/web-office-skins.js +298 -0
- package/dist/web-office-skins.js.map +1 -0
- package/dist/web-office-store.d.ts +8 -2
- package/dist/web-office-store.js +37 -7
- package/dist/web-office-store.js.map +1 -1
- package/dist/web-settings-patch.d.ts +6 -0
- package/dist/web-settings-patch.js +61 -4
- package/dist/web-settings-patch.js.map +1 -1
- package/dist/web-shell-routes.d.ts +4 -2
- package/dist/web-shell-routes.js +17 -7
- package/dist/web-shell-routes.js.map +1 -1
- package/dist/working-recovery.d.ts +104 -0
- package/dist/working-recovery.js +263 -0
- package/dist/working-recovery.js.map +1 -0
- package/dist/working-state-journal.d.ts +70 -0
- package/dist/working-state-journal.js +106 -0
- package/dist/working-state-journal.js.map +1 -0
- package/package.json +1 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { WorkingRecoveryCoordinator, WORKING_RECOVERY_KIND, buildWorkingRecoveryPrompt } from "./working-recovery.js";
|
|
1
2
|
import { PUBLIC_TUNNEL_TOKEN_ENV, PUBLIC_TUNNEL_TOKEN_HINT, normalizePublicTunnelToken } from "./public-tunnel-config.js";
|
|
2
3
|
import { humanOwnerObjectId, webHumanOrigin } from "./human-delivery-receipts.js";
|
|
3
4
|
import { SettingsBaselines } from "./settings-baseline.js";
|
|
@@ -158,6 +159,7 @@ import { agendVersion } from "./agent-whats-new.js";
|
|
|
158
159
|
import { OwnerCssStore } from "./web-owner-css-store.js";
|
|
159
160
|
import { ThemeStore } from "./web-theme-store.js";
|
|
160
161
|
import { OfficeStore, OfficeWriteLimiter } from "./web-office-store.js";
|
|
162
|
+
import { OfficeSkinStore } from "./web-office-skins.js";
|
|
161
163
|
import { WebAppearance } from "./web-appearance.js";
|
|
162
164
|
import { WebEffectLibrary } from "./web-effect-library.js";
|
|
163
165
|
import { WebEffectTrials } from "./web-effect-trials.js";
|
|
@@ -217,6 +219,8 @@ import { assertExplicitInstanceRemoval } from "./instance-removal.js";
|
|
|
217
219
|
import { KNOWN_BACKENDS, validateFleetConfig } from "./config-validator.js";
|
|
218
220
|
import { isRemovedBackend, removedBackendMessage } from "./backend/removed.js";
|
|
219
221
|
import { presentationState, interactionSummary, sameInteractionOwner } from "./interaction-observation.js";
|
|
222
|
+
import { describeSelfOom } from "./session-oom.js";
|
|
223
|
+
import { assertNoSessionOomStop, readSessionOomStop, captureSessionOomRetry, clearSessionOomStopForRetry, SessionOomStoppedError, sessionOomStopHint } from "./session-oom-stop.js";
|
|
220
224
|
import { idleGateTimeoutLog } from "./idle-gate-log.js";
|
|
221
225
|
import { DEFAULT_STUCK_TIMEOUT_MS, readLastInboundAt } from "./daemon.js";
|
|
222
226
|
import { clearPausedMarker, readPausedAt, readPauseReason, writePausedMarker } from "./pause-marker.js";
|
|
@@ -460,6 +464,7 @@ const EFFORT_SELECT_CALLBACK_PREFIX = "effort-select:";
|
|
|
460
464
|
const BACKEND_SELECT_CALLBACK_PREFIX = "backend-select:";
|
|
461
465
|
const INTERACTIVE_ASSIST_CALLBACK_PREFIX = "interactive-assist:";
|
|
462
466
|
const EXIT_RESTART_CALLBACK_PREFIX = "exit-restart:";
|
|
467
|
+
const OOM_RETRY_CALLBACK_PREFIX = "oom-retry:";
|
|
463
468
|
/** #1386: how long a paused instance's marker reason is reused before it is read again (transitions drop it at once). */
|
|
464
469
|
const NEEDS_PAUSE_TTL_MS = 10_000;
|
|
465
470
|
const HANG_CALLBACK_PREFIX = "hang:";
|
|
@@ -471,7 +476,7 @@ const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
|
|
|
471
476
|
* group's approval, tips, and the per-user /model and /effort menus (which are not nonce entries at all).
|
|
472
477
|
*/
|
|
473
478
|
const WEB_MIRRORED_PROMPT_PREFIXES = new Set([
|
|
474
|
-
HANG_CALLBACK_PREFIX, EXIT_RESTART_CALLBACK_PREFIX, INTERACTIVE_ASSIST_CALLBACK_PREFIX,
|
|
479
|
+
HANG_CALLBACK_PREFIX, EXIT_RESTART_CALLBACK_PREFIX, OOM_RETRY_CALLBACK_PREFIX, INTERACTIVE_ASSIST_CALLBACK_PREFIX,
|
|
475
480
|
]);
|
|
476
481
|
/**
|
|
477
482
|
* Where a reply goes on a fleet with no chat platform (web track C4): it is "sent" by being shown in the
|
|
@@ -602,6 +607,89 @@ export class FleetManager {
|
|
|
602
607
|
managerBootId = randomUUID();
|
|
603
608
|
deliveryOutbox = null;
|
|
604
609
|
deliveryOutboxRecovered = false;
|
|
610
|
+
workingRecovery = null;
|
|
611
|
+
workingRecoveryTimer = null;
|
|
612
|
+
workingRecoveryCause = "unknown_interruption";
|
|
613
|
+
recoveryEnabled(name) {
|
|
614
|
+
const own = this.fleetConfig?.instances[name]?.resume_interrupted_work;
|
|
615
|
+
return own ?? this.fleetConfig?.defaults?.resume_interrupted_work ?? true;
|
|
616
|
+
}
|
|
617
|
+
initializeWorkingRecovery(outbox) {
|
|
618
|
+
const store = outbox.workingRecovery;
|
|
619
|
+
this.workingRecoveryCause = store.beginRun(this.managerBootId, process.pid, Date.now());
|
|
620
|
+
this.workingRecovery = new WorkingRecoveryCoordinator({
|
|
621
|
+
store, now: () => performance.now(), active: () => this.startupComplete && !this.shuttingDown && outbox.isOpen,
|
|
622
|
+
pressure: () => {
|
|
623
|
+
const p = this.memoryPressure.snapshot();
|
|
624
|
+
return p.sampledAt === null || Date.now() - p.sampledAt > 60_000 ? { ...p, level: "unknown" } : p;
|
|
625
|
+
}, storm: () => this.stormWindow.isActive(),
|
|
626
|
+
enabled: name => this.recoveryEnabled(name),
|
|
627
|
+
live: name => {
|
|
628
|
+
const daemon = this.daemons.get(name);
|
|
629
|
+
if (this.sessionOomStopped(name))
|
|
630
|
+
return { owner: "", configuration: "", sessionId: null, idle: false,
|
|
631
|
+
paused: false, stopped: true, legacyPending: false, legacyCovered: false, nativeWork: false, cancelled: false, background: null };
|
|
632
|
+
return daemon?.getWorkingRecoveryState?.() ?? null;
|
|
633
|
+
},
|
|
634
|
+
card: id => this.scheduler?.db.board.findCard(id) ?? (this.scheduler ? undefined : null),
|
|
635
|
+
laneFree: name => !this.isInstanceRestarting(name) && !this.activeDurableTargets.has(name)
|
|
636
|
+
&& outbox.countForTarget(name, ["queued", "retry_wait", "delivering", "submission_started", "reconciliation_pending"]) === 0,
|
|
637
|
+
admit: (episode, tasks, advisory) => outbox.admitWorkingRecovery(episode, buildWorkingRecoveryPrompt(tasks, this.workingRecoveryCause, advisory), advisory),
|
|
638
|
+
summary: (group, rows) => {
|
|
639
|
+
const general = this.findGeneralInstance();
|
|
640
|
+
if (!general || !this.daemons.has(general))
|
|
641
|
+
return;
|
|
642
|
+
// Our own informational notice must not supersede General's recovery
|
|
643
|
+
// as unrelated pending work before it crosses the input boundary.
|
|
644
|
+
const generalEpisode = store.get(general);
|
|
645
|
+
if (generalEpisode?.outcome === "admitted" && generalEpisode.deliveryId) {
|
|
646
|
+
const state = outbox.get(generalEpisode.deliveryId)?.state;
|
|
647
|
+
if (!state || !["delivered", "uncertain", "failed", "cancelled"].includes(state))
|
|
648
|
+
return;
|
|
649
|
+
}
|
|
650
|
+
const deferredAdvisories = rows.filter(row => row.outcome === "deferred"
|
|
651
|
+
&& (this.daemons.get(row.instance)?.getWorkingRecoveryState?.().background ?? null) === null)
|
|
652
|
+
.map(row => `${row.instance} (${row.reason})`);
|
|
653
|
+
store.summary(group, () => outbox.admitSystemNotice(general, `working-recovery-summary:${group}`, `[system:working-recovery-summary] Observed cause: unknown interruption${this.workingRecoveryCause === "unknown_interruption" ? "" : `; prior requested action: ${this.workingRecoveryCause}`}. ${rows.length} interrupted instances; continuation queued ${rows.filter(r => r.outcome === "admitted").length}, excluded ${rows.filter(r => r.outcome === "excluded").length}, still deferred ${rows.filter(r => r.outcome === "deferred").length}. ${rows.filter(r => r.reason === "advisory_background_unknown").length ? `Advisory, background unknown: ${rows.filter(r => r.reason === "advisory_background_unknown").map(r => r.instance).join(", ")}. ` : ""}${deferredAdvisories.length ? `Deferred advisory candidates (background unknown; all other gates still apply): ${deferredAdvisories.join(", ")}. ` : ""}Details are in the Log. Deferred work remains eligible; no action or confirmation is requested.`));
|
|
654
|
+
},
|
|
655
|
+
log: (episode, outcome, reason) => {
|
|
656
|
+
const requestedAction = this.workingRecoveryCause === "unknown_interruption" ? null : this.workingRecoveryCause;
|
|
657
|
+
this.logger.info({ name: episode.instance, episode: episode.key, outcome, reason,
|
|
658
|
+
observedCause: "unknown_interruption", requestedAction }, "Interrupted work recovery decision");
|
|
659
|
+
this.eventLog?.insert(episode.instance, "working_state", {
|
|
660
|
+
message: `Interrupted work: ${outcome} (${reason}); observed cause unknown_interruption${requestedAction ? `; prior requested action ${requestedAction}` : ""}.`,
|
|
661
|
+
reason, outcome, observedCause: "unknown_interruption", requestedAction
|
|
662
|
+
});
|
|
663
|
+
},
|
|
664
|
+
});
|
|
665
|
+
if (this.workingRecoveryTimer)
|
|
666
|
+
clearInterval(this.workingRecoveryTimer);
|
|
667
|
+
this.workingRecoveryTimer = setInterval(() => {
|
|
668
|
+
if (!outbox.isOpen || this.shuttingDown)
|
|
669
|
+
return;
|
|
670
|
+
try {
|
|
671
|
+
this.workingRecovery?.tick();
|
|
672
|
+
this.scheduleDeliveryOutboxPump();
|
|
673
|
+
}
|
|
674
|
+
catch {
|
|
675
|
+
this.logger.warn("Working recovery deferred: prerequisites or durable store unavailable");
|
|
676
|
+
}
|
|
677
|
+
}, 1_000);
|
|
678
|
+
this.workingRecoveryTimer.unref?.();
|
|
679
|
+
}
|
|
680
|
+
recoveryDeliveryEligible(row, writing = false) {
|
|
681
|
+
if (row.kind !== WORKING_RECOVERY_KIND)
|
|
682
|
+
return true;
|
|
683
|
+
const outbox = this.deliveryOutbox, episode = outbox?.workingRecovery.get(row.targetInstance);
|
|
684
|
+
if (!episode || episode.deliveryId !== row.deliveryId
|
|
685
|
+
|| outbox.hasOtherPendingWork(row.targetInstance, row.deliveryId)
|
|
686
|
+
|| this.workingRecovery?.verdict(episode).outcome === "excluded") {
|
|
687
|
+
outbox?.withdrawWorkingRecovery(row.deliveryId, "recovery superseded or no longer authorized before input");
|
|
688
|
+
return false;
|
|
689
|
+
}
|
|
690
|
+
return (writing ? this.workingRecovery?.accepts(row.targetInstance, episode.key)
|
|
691
|
+
: this.workingRecovery?.canDispatch(row.targetInstance, episode.key)) === true;
|
|
692
|
+
}
|
|
605
693
|
/** Phase 2b: wakes paused targets for queued durable work (delivery_worker ≠ off). */
|
|
606
694
|
wakeCoordinator = null;
|
|
607
695
|
/**
|
|
@@ -759,6 +847,8 @@ export class FleetManager {
|
|
|
759
847
|
* backend outage during a post-update herd). instanceName → pending retry.
|
|
760
848
|
*/
|
|
761
849
|
startupRetries = new Map();
|
|
850
|
+
/** #2484: instances whose automatic startup retry is running now (between its timer and its outcome). */
|
|
851
|
+
startupRetriesRunning = new Set();
|
|
762
852
|
/** Bumped by every explicit stop (incl. the stop half of a restart); fences the outage hand-off. */
|
|
763
853
|
explicitStopGeneration = new Map();
|
|
764
854
|
/** instanceName → outage hand-off currently executing; a repeat joins it. */
|
|
@@ -1049,7 +1139,12 @@ export class FleetManager {
|
|
|
1049
1139
|
}
|
|
1050
1140
|
officeStoreInstance;
|
|
1051
1141
|
/** Fleet → Org chart → Office: the offices people made (web-offices.json, beside web-themes.json). */
|
|
1052
|
-
get officeStore() { return (this.officeStoreInstance ??= new OfficeStore(this.dataDir)); }
|
|
1142
|
+
get officeStore() { return (this.officeStoreInstance ??= new OfficeStore(this.dataDir, id => this.officeSkins.pack(id))); }
|
|
1143
|
+
officeSkinsInstance;
|
|
1144
|
+
/** #2513: the Office view's uploaded skin packs (<AGEND_HOME>/web-office-skins/). */
|
|
1145
|
+
get officeSkins() { return (this.officeSkinsInstance ??= new OfficeSkinStore(this.dataDir)); }
|
|
1146
|
+
/** How often a person may upload or delete a skin pack; memory only, as the offices' limiter. */
|
|
1147
|
+
officeSkinWrites = new OfficeWriteLimiter(undefined, { perWindow: 10, windowMs: 60_000 });
|
|
1053
1148
|
/** How often a person may save the offices (PUT /ui/offices); memory only, as the appearance limiter. */
|
|
1054
1149
|
officeWrites = new OfficeWriteLimiter();
|
|
1055
1150
|
/** #1629: the theme tools, a person's reset and lock, and the one appearance rate limiter (shared with effects). */
|
|
@@ -1792,7 +1887,7 @@ export class FleetManager {
|
|
|
1792
1887
|
...(web && by.guest ? { web_guest: "1" } : {}), // #1920: "[user:… via web (guest), id:…]", as /ui/send
|
|
1793
1888
|
...reactions.meta,
|
|
1794
1889
|
},
|
|
1795
|
-
}, by.humanOrigin, button.label)) !== false;
|
|
1890
|
+
}, by.humanOrigin, button.label, this.startRetryHoldFor(set.instance, inbound))) !== false;
|
|
1796
1891
|
}
|
|
1797
1892
|
finally {
|
|
1798
1893
|
// Both false and throw reopen the choice in the controller. Finish the
|
|
@@ -1841,8 +1936,19 @@ export class FleetManager {
|
|
|
1841
1936
|
}
|
|
1842
1937
|
this.deliveryOutbox = outbox;
|
|
1843
1938
|
this.deliveryOutboxRecovered = true;
|
|
1939
|
+
this.initializeWorkingRecovery(outbox);
|
|
1940
|
+
outbox.on("working_references", (names) => {
|
|
1941
|
+
for (const name of names)
|
|
1942
|
+
this.observeWorkingState(name, "obligations");
|
|
1943
|
+
});
|
|
1844
1944
|
outbox.on("admitted", () => { this.scheduleDeliveryOutboxPump(); this.wakeCoordinator?.kick(); });
|
|
1845
1945
|
outbox.on("state", (event) => {
|
|
1946
|
+
if (event.deliveryId) {
|
|
1947
|
+
const row = outbox.get(event.deliveryId);
|
|
1948
|
+
if (row)
|
|
1949
|
+
for (const name of new Set([row.sourceInstance, row.targetInstance]))
|
|
1950
|
+
this.observeWorkingState(name, "obligations");
|
|
1951
|
+
}
|
|
1846
1952
|
this.scheduleDeliveryOutboxPump();
|
|
1847
1953
|
this.wakeCoordinator?.kick();
|
|
1848
1954
|
if (event.deliveryId && event.state === "delivered") {
|
|
@@ -2001,8 +2107,19 @@ export class FleetManager {
|
|
|
2001
2107
|
: item),
|
|
2002
2108
|
};
|
|
2003
2109
|
}
|
|
2110
|
+
/** A terminal OOM stop is not an auto-wakeable pause, including after a fleet restart. */
|
|
2111
|
+
sessionOomStopped(name) {
|
|
2112
|
+
return this.daemons.get(name)?.isSessionOomStopped?.() === true || readSessionOomStop(this.getInstanceDir(name)).blocked;
|
|
2113
|
+
}
|
|
2114
|
+
sessionOomStopReason(name) {
|
|
2115
|
+
const state = readSessionOomStop(this.getInstanceDir(name));
|
|
2116
|
+
return state.blocked ? sessionOomStopHint(state)
|
|
2117
|
+
: this.daemons.get(name)?.notAcceptingReason?.() ?? sessionOomStopHint();
|
|
2118
|
+
}
|
|
2004
2119
|
/** #1641: why a target is not taking queued work right now, or null when nothing known holds it. */
|
|
2005
2120
|
deliveryWaitingReason(target) {
|
|
2121
|
+
if (this.sessionOomStopped(target))
|
|
2122
|
+
return this.sessionOomStopReason(target);
|
|
2006
2123
|
if (this.lifecycle.isPaused(target))
|
|
2007
2124
|
return t("delivery.hold_state", "paused");
|
|
2008
2125
|
return this.daemons.get(target)?.notAcceptingReason?.() ?? null;
|
|
@@ -2015,6 +2132,9 @@ export class FleetManager {
|
|
|
2015
2132
|
}
|
|
2016
2133
|
/** Called after a replacement Daemon object is installed in lifecycle.daemons. */
|
|
2017
2134
|
onDaemonReady(name, daemonBootId) {
|
|
2135
|
+
const daemon = this.daemons.get(name);
|
|
2136
|
+
if (daemon?.bootId === daemonBootId)
|
|
2137
|
+
this.listenForWorkingState(name, daemon);
|
|
2018
2138
|
const outbox = this.deliveryOutbox;
|
|
2019
2139
|
if (!outbox)
|
|
2020
2140
|
return;
|
|
@@ -2038,6 +2158,86 @@ export class FleetManager {
|
|
|
2038
2158
|
}
|
|
2039
2159
|
this.scheduleDeliveryOutboxPump();
|
|
2040
2160
|
}
|
|
2161
|
+
workingStateListeners;
|
|
2162
|
+
workingStateErrors;
|
|
2163
|
+
/** Attach only to the published owner. Nothing here sends or resumes work. */
|
|
2164
|
+
listenForWorkingState(name, daemon) {
|
|
2165
|
+
if (typeof daemon.getWorkingStateObservation !== "function")
|
|
2166
|
+
return;
|
|
2167
|
+
this.workingStateListeners ??= new WeakSet();
|
|
2168
|
+
if (this.workingStateListeners.has(daemon))
|
|
2169
|
+
return;
|
|
2170
|
+
this.workingStateListeners.add(daemon);
|
|
2171
|
+
daemon.on("working_state", (event) => {
|
|
2172
|
+
try {
|
|
2173
|
+
if (this.daemons.get(name) !== daemon || !sameInteractionOwner(event.observation.owner, daemon.getInteractionSnapshot().owner))
|
|
2174
|
+
return;
|
|
2175
|
+
this.observeWorkingState(name, event.reason, event.observation);
|
|
2176
|
+
}
|
|
2177
|
+
catch { /* A malformed/stale observation cannot affect the daemon. */ }
|
|
2178
|
+
});
|
|
2179
|
+
this.observeWorkingState(name, "published");
|
|
2180
|
+
}
|
|
2181
|
+
observeWorkingState(name, reason, supplied) {
|
|
2182
|
+
const outbox = this.deliveryOutbox, daemon = this.daemons.get(name);
|
|
2183
|
+
if (!outbox || !daemon || typeof daemon.getWorkingStateObservation !== "function")
|
|
2184
|
+
return;
|
|
2185
|
+
try {
|
|
2186
|
+
const observation = supplied ?? daemon.getWorkingStateObservation();
|
|
2187
|
+
if (!sameInteractionOwner(observation.owner, daemon.getInteractionSnapshot().owner))
|
|
2188
|
+
return;
|
|
2189
|
+
const references = [];
|
|
2190
|
+
const unavailable = [];
|
|
2191
|
+
let totalReferences = 0;
|
|
2192
|
+
const board = this.scheduler?.db.board;
|
|
2193
|
+
if (!board)
|
|
2194
|
+
unavailable.push("tasks");
|
|
2195
|
+
else
|
|
2196
|
+
try {
|
|
2197
|
+
const tasks = board.listCards({ assignee: name, status: ["claimed", "blocked", "review"], limit: 16 });
|
|
2198
|
+
totalReferences += tasks.total;
|
|
2199
|
+
for (const card of tasks.cards)
|
|
2200
|
+
references.push({ kind: "task", id: card.uuid, version: card.version,
|
|
2201
|
+
status: card.status });
|
|
2202
|
+
}
|
|
2203
|
+
catch {
|
|
2204
|
+
unavailable.push("tasks");
|
|
2205
|
+
}
|
|
2206
|
+
try {
|
|
2207
|
+
const obligations = outbox.workingObligations(name);
|
|
2208
|
+
references.push(...obligations.references);
|
|
2209
|
+
totalReferences += obligations.total;
|
|
2210
|
+
}
|
|
2211
|
+
catch {
|
|
2212
|
+
unavailable.push("obligations");
|
|
2213
|
+
}
|
|
2214
|
+
const entry = outbox.observeWorkingState(name, { version: 1, reason, observedAt: Date.now(), observation,
|
|
2215
|
+
lastActivityAt: this.lastActivity.get(name) ?? null, references, totalReferences, unavailable });
|
|
2216
|
+
if (!entry)
|
|
2217
|
+
return;
|
|
2218
|
+
if (reason === "cancelled")
|
|
2219
|
+
outbox.workingRecovery.cancel(name, observation.configuration, observation.sessionId);
|
|
2220
|
+
if (entry.prior)
|
|
2221
|
+
outbox.workingRecovery.discover(name, entry.prior);
|
|
2222
|
+
const state = entry.current.observation.executionState ?? "unknown";
|
|
2223
|
+
const counts = { tasks: entry.current.references.filter(r => r.kind === "task").length,
|
|
2224
|
+
owed: entry.current.references.filter(r => r.kind === "owed").length,
|
|
2225
|
+
waiting: entry.current.references.filter(r => r.kind === "waiting").length,
|
|
2226
|
+
background: entry.current.observation.backgroundWork,
|
|
2227
|
+
total: entry.current.totalReferences, unavailable: entry.current.unavailable };
|
|
2228
|
+
this.logger.info({ name, state, reason, revision: entry.revision, ...counts }, "Working-state journal observed");
|
|
2229
|
+
this.eventLog?.insert(name, "working_state", { state, reason, revision: entry.revision, ...counts,
|
|
2230
|
+
message: `Observed ${state}; tasks ${counts.tasks}, replies owed ${counts.owed}, waiting ${counts.waiting}; ${counts.total} references${counts.background === true ? "; background work running" : ""}${unavailable.length ? "; some sources unavailable" : ""}. Journal only; no recovery action.` });
|
|
2231
|
+
this.emitSseEvent("steps_tick", { instance: name });
|
|
2232
|
+
}
|
|
2233
|
+
catch {
|
|
2234
|
+
this.workingStateErrors ??= new Set();
|
|
2235
|
+
if (!this.workingStateErrors.has(name)) {
|
|
2236
|
+
this.workingStateErrors.add(name);
|
|
2237
|
+
this.logger.warn({ name }, "Could not persist working-state observation; delivery and restart behavior unchanged");
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
}
|
|
2041
2241
|
/**
|
|
2042
2242
|
* #926: the two reply-obligation safety nets. Idle owners with an unanswered
|
|
2043
2243
|
* request get one reminder (after the grace); requesters get one overdue
|
|
@@ -2112,7 +2312,7 @@ export class FleetManager {
|
|
|
2112
2312
|
// Phase 2c: a lane a queue worker owns is never claimed here.
|
|
2113
2313
|
target => this.queueWorkers.has(target) || this.isInstanceRestarting(target) || this.wakeCoordinator?.blocksClaim(target)
|
|
2114
2314
|
? null
|
|
2115
|
-
: this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets);
|
|
2315
|
+
: this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets, row => this.recoveryDeliveryEligible(row));
|
|
2116
2316
|
if (!claimed)
|
|
2117
2317
|
break;
|
|
2118
2318
|
this.activeDurableTargets.add(claimed.targetInstance);
|
|
@@ -2174,7 +2374,7 @@ export class FleetManager {
|
|
|
2174
2374
|
available: () => !this.shuttingDown && this.deliveryOutbox?.isOpen === true,
|
|
2175
2375
|
blocked: () => this.isInstanceRestarting(target) || this.wakeCoordinator?.blocksClaim(target) === true,
|
|
2176
2376
|
daemonBootId: () => this.daemons.get(target)?.bootId ?? null,
|
|
2177
|
-
claim: bootId => this.deliveryOutbox?.claimNext(this.managerBootId, candidate => candidate === target ? bootId : null, new Set()),
|
|
2377
|
+
claim: bootId => this.deliveryOutbox?.claimNext(this.managerBootId, candidate => candidate === target ? bootId : null, new Set(), row => this.recoveryDeliveryEligible(row)),
|
|
2178
2378
|
tryAcquireBudget: () => {
|
|
2179
2379
|
if (this.activeDurableTargets.size >= 8)
|
|
2180
2380
|
return false;
|
|
@@ -2215,7 +2415,24 @@ export class FleetManager {
|
|
|
2215
2415
|
// including time spent waiting for a handoff/idle gate to return.
|
|
2216
2416
|
try {
|
|
2217
2417
|
let sent;
|
|
2218
|
-
if (claimed.kind ===
|
|
2418
|
+
if (claimed.kind === WORKING_RECOVERY_KIND) {
|
|
2419
|
+
const daemon = this.daemons.get(target), key = claimed.sourceKey.replace(/^working-recovery:/, "");
|
|
2420
|
+
if (!daemon || typeof payload.content !== "string")
|
|
2421
|
+
throw new Error("Working recovery owner unavailable");
|
|
2422
|
+
const release = this.workingRecovery?.reserveDispatch(target, key);
|
|
2423
|
+
if (!release) {
|
|
2424
|
+
outbox.retryBeforeBegin(claimed.deliveryId, daemon.bootId, claimed.attemptNo, "working recovery pace or prerequisites pending", 5_000);
|
|
2425
|
+
return;
|
|
2426
|
+
}
|
|
2427
|
+
try {
|
|
2428
|
+
await daemon.deliverWorkingRecovery(payload.content, claimed, () => this.daemons.get(target) === daemon && this.recoveryDeliveryEligible(claimed, true));
|
|
2429
|
+
}
|
|
2430
|
+
finally {
|
|
2431
|
+
release();
|
|
2432
|
+
}
|
|
2433
|
+
sent = true;
|
|
2434
|
+
}
|
|
2435
|
+
else if (claimed.kind === "raw_paste") {
|
|
2219
2436
|
if (payload.type !== "raw_paste" || typeof payload.content !== "string") {
|
|
2220
2437
|
throw new Error("Durable raw_paste row has an invalid payload");
|
|
2221
2438
|
}
|
|
@@ -2899,6 +3116,7 @@ export class FleetManager {
|
|
|
2899
3116
|
}
|
|
2900
3117
|
/** SIGUSR1: full process reload. Names who asked (#1120). */
|
|
2901
3118
|
onFullRestartSignal() {
|
|
3119
|
+
this.recordWorkingRestartIntent("manual_restart");
|
|
2902
3120
|
this.logger.info(`Received SIGUSR1, initiating full restart (process reload)... ${describeSignalSource(this.dataDir, "SIGUSR1")}`);
|
|
2903
3121
|
this.gracefulShutdownForReload()
|
|
2904
3122
|
.then(() => {
|
|
@@ -3725,7 +3943,7 @@ export class FleetManager {
|
|
|
3725
3943
|
finish(false);
|
|
3726
3944
|
});
|
|
3727
3945
|
}
|
|
3728
|
-
async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch, noInlineWake = false, stillCurrent) {
|
|
3946
|
+
async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch, noInlineWake = false, stillCurrent, onStartRetryHold) {
|
|
3729
3947
|
if (this.shuttingDown || !this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
|
|
3730
3948
|
return false;
|
|
3731
3949
|
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
@@ -3764,7 +3982,7 @@ export class FleetManager {
|
|
|
3764
3982
|
const log = idleGateTimeoutLog(instanceName, timeoutMs, this.instanceStateCache.get(instanceName), Date.now());
|
|
3765
3983
|
this.logger[log.level](log.fields, log.message);
|
|
3766
3984
|
}
|
|
3767
|
-
const sent = await this.sendWhenConnected(instanceName, payload, deliveryEpoch, stillCurrent);
|
|
3985
|
+
const sent = await this.sendWhenConnected(instanceName, payload, deliveryEpoch, stillCurrent, onStartRetryHold);
|
|
3768
3986
|
if (!sent)
|
|
3769
3987
|
return false;
|
|
3770
3988
|
if (this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent)) {
|
|
@@ -3793,8 +4011,11 @@ export class FleetManager {
|
|
|
3793
4011
|
* Ordering is preserved by serialising behind any waiter already queued for this
|
|
3794
4012
|
* instance, *including* when the socket happens to be up: otherwise a message
|
|
3795
4013
|
* arriving after the reconnect could overtake one that has been waiting for it.
|
|
4014
|
+
*
|
|
4015
|
+
* `onStartRetryHold` (#2484, a person's message): an instance in its own start-retry backoff is waited for until that
|
|
4016
|
+
* retry brings it up, however long the backoff, and fails only once the retries give up.
|
|
3796
4017
|
*/
|
|
3797
|
-
async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent) {
|
|
4018
|
+
async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent, onStartRetryHold) {
|
|
3798
4019
|
if (!this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
|
|
3799
4020
|
return false;
|
|
3800
4021
|
const queued = this.ipcWaitTails.get(instanceName);
|
|
@@ -3808,7 +4029,7 @@ export class FleetManager {
|
|
|
3808
4029
|
.then(() => {
|
|
3809
4030
|
if (!this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
|
|
3810
4031
|
return false;
|
|
3811
|
-
return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch, stillCurrent);
|
|
4032
|
+
return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch, stillCurrent, onStartRetryHold);
|
|
3812
4033
|
});
|
|
3813
4034
|
// The chain stores a settled-either-way promise so one failed delivery cannot
|
|
3814
4035
|
// wedge every later one, and so `queued` above is safe to await unguarded.
|
|
@@ -3826,9 +4047,10 @@ export class FleetManager {
|
|
|
3826
4047
|
}
|
|
3827
4048
|
}
|
|
3828
4049
|
/** Poll for the instance's IPC to come back, then send. Throws if it does not. */
|
|
3829
|
-
async sendAfterIpcReturns(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent) {
|
|
3830
|
-
|
|
4050
|
+
async sendAfterIpcReturns(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent, onStartRetryHold) {
|
|
4051
|
+
let deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
|
|
3831
4052
|
let warned = false;
|
|
4053
|
+
let held = false;
|
|
3832
4054
|
for (;;) {
|
|
3833
4055
|
if (!this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
|
|
3834
4056
|
return false;
|
|
@@ -3837,6 +4059,22 @@ export class FleetManager {
|
|
|
3837
4059
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
3838
4060
|
if (ipc?.connected && ipc.send(this.bindHumanReceiptPayload(instanceName, payload)))
|
|
3839
4061
|
return true;
|
|
4062
|
+
// #2484: while the instance waits for (or runs) its own start retry, the grace runs from now — so the message is
|
|
4063
|
+
// delivered once a retry brings it up, and fails only after the retries stop (they gave up, or a person stopped
|
|
4064
|
+
// or paused it) and a last grace passes without it.
|
|
4065
|
+
if (onStartRetryHold && this.inStartRetryBackoff(instanceName)) {
|
|
4066
|
+
deadline = Math.max(deadline, Date.now() + IPC_RECONNECT_GRACE_MS);
|
|
4067
|
+
if (!held) {
|
|
4068
|
+
held = true;
|
|
4069
|
+
this.logger.info({ instanceName, retry: this.startupRetries.get(instanceName)?.attempt }, "Instance is waiting for its start retry — holding the message until it is up");
|
|
4070
|
+
try {
|
|
4071
|
+
onStartRetryHold();
|
|
4072
|
+
}
|
|
4073
|
+
catch (err) {
|
|
4074
|
+
this.logger.debug({ err, instanceName }, "Start-retry hold notice failed");
|
|
4075
|
+
}
|
|
4076
|
+
}
|
|
4077
|
+
}
|
|
3840
4078
|
if (Date.now() >= deadline) {
|
|
3841
4079
|
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
3842
4080
|
}
|
|
@@ -4222,6 +4460,18 @@ export class FleetManager {
|
|
|
4222
4460
|
this.deliveryOutbox?.markHumanReceiptUncertain(id, instanceName);
|
|
4223
4461
|
}
|
|
4224
4462
|
}
|
|
4463
|
+
/**
|
|
4464
|
+
* #2484: a person's message waits for its instance's own start retry (DeliveryOptions.onStartRetryHold), shown as
|
|
4465
|
+
* queued on the message meanwhile. A ClassicBot room message (no `msg`) has no delivery-status reactions: it only waits.
|
|
4466
|
+
*/
|
|
4467
|
+
startRetryHoldFor(instanceName, msg) {
|
|
4468
|
+
return {
|
|
4469
|
+
onStartRetryHold: () => {
|
|
4470
|
+
if (msg?.chatId && msg.messageId)
|
|
4471
|
+
this.reactMessageStatus(instanceName, msg.chatId, msg.messageId, "queued", msg.threadId || undefined);
|
|
4472
|
+
},
|
|
4473
|
+
};
|
|
4474
|
+
}
|
|
4225
4475
|
/** Synchronous steer/btw handoff, using the same evidence and owner binding. */
|
|
4226
4476
|
sendHumanControl(instanceName, payload, origin, originalBody) {
|
|
4227
4477
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
@@ -4262,7 +4512,7 @@ export class FleetManager {
|
|
|
4262
4512
|
return false;
|
|
4263
4513
|
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
4264
4514
|
}
|
|
4265
|
-
const sent = await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch, options.stillCurrent);
|
|
4515
|
+
const sent = await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch, options.stillCurrent, options.onStartRetryHold);
|
|
4266
4516
|
if (!sent)
|
|
4267
4517
|
return false;
|
|
4268
4518
|
// A cross-instance item arriving before the daemon observes this turn as
|
|
@@ -4273,7 +4523,7 @@ export class FleetManager {
|
|
|
4273
4523
|
return true;
|
|
4274
4524
|
}
|
|
4275
4525
|
const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
|
|
4276
|
-
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch, options.noInlineWake === true, options.stillCurrent));
|
|
4526
|
+
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch, options.noInlineWake === true, options.stillCurrent, options.onStartRetryHold));
|
|
4277
4527
|
this.idleGatedDeliveryTails.set(instanceName, delivery);
|
|
4278
4528
|
try {
|
|
4279
4529
|
return await delivery;
|
|
@@ -4729,8 +4979,8 @@ export class FleetManager {
|
|
|
4729
4979
|
return new WakeCoordinator({
|
|
4730
4980
|
mode: target => this.deliveryWorkerMode(target),
|
|
4731
4981
|
available: () => !this.shuttingDown && this.deliveryOutbox?.isOpen === true,
|
|
4732
|
-
listPending: () => this.deliveryOutbox?.listPending() ?? [],
|
|
4733
|
-
isPaused: target => this.lifecycle.isPaused(target),
|
|
4982
|
+
listPending: () => this.deliveryOutbox?.listPending().filter(row => row.kind !== WORKING_RECOVERY_KIND) ?? [],
|
|
4983
|
+
isPaused: target => !this.sessionOomStopped(target) && this.lifecycle.isPaused(target),
|
|
4734
4984
|
pauseReason: target => readPauseReason(this.getInstanceDir(target)),
|
|
4735
4985
|
isRestarting: target => this.isInstanceRestarting(target),
|
|
4736
4986
|
wake: async (target) => {
|
|
@@ -4742,7 +4992,7 @@ export class FleetManager {
|
|
|
4742
4992
|
warmCap: () => this.fleetConfig?.defaults?.warm_cap ?? 0,
|
|
4743
4993
|
warmOverflow: () => this.fleetConfig?.defaults?.warm_overflow ?? DEFAULT_WARM_OVERFLOW,
|
|
4744
4994
|
wokeAt: target => this.lifecycle.wokeAtFor(target),
|
|
4745
|
-
notAcceptingReason: target => this.daemons.get(target)?.notAcceptingReason?.() ?? null,
|
|
4995
|
+
notAcceptingReason: target => this.sessionOomStopped(target) ? this.sessionOomStopReason(target) : this.daemons.get(target)?.notAcceptingReason?.() ?? null,
|
|
4746
4996
|
notifyTarget: (target, text) => { this.notifyInstanceTopic(target, text); },
|
|
4747
4997
|
notifySender: (source, text) => { this.notifyInstanceTopic(source, text); },
|
|
4748
4998
|
kickPump: () => this.scheduleDeliveryOutboxPump(),
|
|
@@ -4797,7 +5047,12 @@ export class FleetManager {
|
|
|
4797
5047
|
* is exactly the old lifecycle wake; otherwise it goes through the shared
|
|
4798
5048
|
* slot admission and joins any wake already in flight for the target.
|
|
4799
5049
|
*/
|
|
4800
|
-
async explicitWake(name, timeoutMs = 30_000) {
|
|
5050
|
+
async explicitWake(name, timeoutMs = 30_000, retryStoppedSession = false) {
|
|
5051
|
+
if (this.sessionOomStopped(name)) {
|
|
5052
|
+
if (!retryStoppedSession)
|
|
5053
|
+
throw new SessionOomStoppedError(false, readSessionOomStop(this.getInstanceDir(name)));
|
|
5054
|
+
return this.restartSingleInstance(name, { explicit: true });
|
|
5055
|
+
}
|
|
4801
5056
|
const token = await this.acquireWakeSlot(name);
|
|
4802
5057
|
try {
|
|
4803
5058
|
await this.lifecycle.wake(name, timeoutMs);
|
|
@@ -4818,7 +5073,7 @@ export class FleetManager {
|
|
|
4818
5073
|
}
|
|
4819
5074
|
async changeInstancePauseState(name, action) {
|
|
4820
5075
|
if (action === "wake") {
|
|
4821
|
-
await this.explicitWake(name, 30_000);
|
|
5076
|
+
await this.explicitWake(name, 30_000, true);
|
|
4822
5077
|
this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
|
|
4823
5078
|
return "awake";
|
|
4824
5079
|
}
|
|
@@ -4994,6 +5249,11 @@ export class FleetManager {
|
|
|
4994
5249
|
/** Phase 2a: the caller's transition, for a start made from inside a restart or wake. */
|
|
4995
5250
|
transition, execution) {
|
|
4996
5251
|
execution?.assert();
|
|
5252
|
+
if (this.sessionOomStopped(name)) {
|
|
5253
|
+
if (resumePaused)
|
|
5254
|
+
return this.restartSingleInstance(name, { explicit: true });
|
|
5255
|
+
throw new SessionOomStoppedError(false, readSessionOomStop(this.getInstanceDir(name)));
|
|
5256
|
+
}
|
|
4997
5257
|
// CLI single-instance cold starts bypass startAll. Start diagnostics before
|
|
4998
5258
|
// any lifecycle/wake work; a shutdown must not revive the stopped sampler.
|
|
4999
5259
|
if (!this.shuttingDown)
|
|
@@ -5002,7 +5262,7 @@ export class FleetManager {
|
|
|
5002
5262
|
// into a running instance — harmless, but noisy — or race this start).
|
|
5003
5263
|
this.cancelStartupRetry(name);
|
|
5004
5264
|
if (resumePaused && this.lifecycle.isPaused(name)) {
|
|
5005
|
-
await this.explicitWake(name, 30_000);
|
|
5265
|
+
await this.explicitWake(name, 30_000, true);
|
|
5006
5266
|
// A successful wake clears the persisted pause marker and produces a
|
|
5007
5267
|
// fresh instance_state snapshot. Drop any stale process error left by a
|
|
5008
5268
|
// pre-pause crash so /api/fleet and `agend ls` converge on running.
|
|
@@ -5022,7 +5282,7 @@ export class FleetManager {
|
|
|
5022
5282
|
if (resumePaused) {
|
|
5023
5283
|
const status = this.getInstanceStatus(name);
|
|
5024
5284
|
if (status === "crashed" || status === "stopped") {
|
|
5025
|
-
await this.restartSingleInstance(name);
|
|
5285
|
+
await this.restartSingleInstance(name, { explicit: true });
|
|
5026
5286
|
return;
|
|
5027
5287
|
}
|
|
5028
5288
|
}
|
|
@@ -5197,6 +5457,8 @@ export class FleetManager {
|
|
|
5197
5457
|
// outcome. Retrying would start an instance the operator just stopped.
|
|
5198
5458
|
if (err instanceof SupersededStartError)
|
|
5199
5459
|
return false;
|
|
5460
|
+
if (err instanceof SessionOomStoppedError)
|
|
5461
|
+
await this.notifySessionOomStop(name);
|
|
5200
5462
|
if (this.stopOnUnsupportedCli(name, err, what))
|
|
5201
5463
|
return false;
|
|
5202
5464
|
this.logger.error({ err, name }, `Failed to start ${what}`);
|
|
@@ -5217,6 +5479,11 @@ export class FleetManager {
|
|
|
5217
5479
|
scheduleStartupRetry(name, attempt) {
|
|
5218
5480
|
if (this.shuttingDown)
|
|
5219
5481
|
return;
|
|
5482
|
+
if (this.sessionOomStopped(name)) {
|
|
5483
|
+
this.instanceProcessStatus.set(name, "stopped");
|
|
5484
|
+
this.logger.warn({ name }, this.sessionOomStopReason(name));
|
|
5485
|
+
return;
|
|
5486
|
+
}
|
|
5220
5487
|
if (this.startupRetries.has(name))
|
|
5221
5488
|
return;
|
|
5222
5489
|
const backoff = FleetManager.STARTUP_RETRY_BACKOFF_MS;
|
|
@@ -5244,6 +5511,12 @@ export class FleetManager {
|
|
|
5244
5511
|
this.startupRetries.delete(name);
|
|
5245
5512
|
this.logger.info({ name }, "Pending automatic startup retry cancelled");
|
|
5246
5513
|
}
|
|
5514
|
+
/** #2484: the instance's own start retry is scheduled or running, and no person paused it (a stop cancels the retry). */
|
|
5515
|
+
inStartRetryBackoff(name) {
|
|
5516
|
+
if (this.shuttingDown || this.lifecycle.isPaused(name))
|
|
5517
|
+
return false;
|
|
5518
|
+
return this.startupRetries.has(name) || this.startupRetriesRunning.has(name);
|
|
5519
|
+
}
|
|
5247
5520
|
/** Pending automatic retry, if any (status display / tests). */
|
|
5248
5521
|
pendingStartupRetry(name) {
|
|
5249
5522
|
const pending = this.startupRetries.get(name);
|
|
@@ -5294,6 +5567,8 @@ export class FleetManager {
|
|
|
5294
5567
|
this.startupRetries.delete(name);
|
|
5295
5568
|
if (this.shuttingDown)
|
|
5296
5569
|
return;
|
|
5570
|
+
if (this.sessionOomStopped(name))
|
|
5571
|
+
return;
|
|
5297
5572
|
if (!this.isConfiguredInstance(name))
|
|
5298
5573
|
return; // removed from fleet.yaml / classic channel meanwhile
|
|
5299
5574
|
if (this.lifecycle.isPaused(name) || this.daemons.has(name))
|
|
@@ -5309,6 +5584,7 @@ export class FleetManager {
|
|
|
5309
5584
|
}
|
|
5310
5585
|
const topicMode = this.fleetConfig?.channel?.mode === "topic"
|
|
5311
5586
|
|| !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
|
|
5587
|
+
this.startupRetriesRunning.add(name);
|
|
5312
5588
|
try {
|
|
5313
5589
|
await this.spawnGate.run({
|
|
5314
5590
|
instanceName: name,
|
|
@@ -5330,6 +5606,9 @@ export class FleetManager {
|
|
|
5330
5606
|
this.logger.error({ err, name, attempt: attempt + 1 }, "Automatic startup retry failed");
|
|
5331
5607
|
this.scheduleStartupRetry(name, attempt + 1);
|
|
5332
5608
|
}
|
|
5609
|
+
finally {
|
|
5610
|
+
this.startupRetriesRunning.delete(name);
|
|
5611
|
+
}
|
|
5333
5612
|
}
|
|
5334
5613
|
/** Fleet-topic (fleet.yaml) or ClassicBot (classic channel) — both are retryable; anything else is gone. */
|
|
5335
5614
|
isConfiguredInstance(name) {
|
|
@@ -5655,12 +5934,29 @@ export class FleetManager {
|
|
|
5655
5934
|
}
|
|
5656
5935
|
/** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
|
|
5657
5936
|
async restartSingleInstance(name, opts) {
|
|
5937
|
+
if (opts?.recoveryCurrent && !opts.recoveryCurrent())
|
|
5938
|
+
throw new Error(t("exit.expired", name));
|
|
5939
|
+
const retry = opts?.explicit ? opts.sessionOomRetry ?? captureSessionOomRetry(this.getInstanceDir(name)) : undefined;
|
|
5940
|
+
if (opts?.sessionOomRetry) {
|
|
5941
|
+
const current = captureSessionOomRetry(this.getInstanceDir(name));
|
|
5942
|
+
// An unclear-cause stop intentionally has no tombstone. Only its live
|
|
5943
|
+
// launch fence can authorize an absent-marker receipt; positive stops
|
|
5944
|
+
// still require the offered durable revision.
|
|
5945
|
+
if ((!retry?.revision && !opts.recoveryCurrent) || current.revision !== retry?.revision || current.legacyRevision !== retry?.legacyRevision)
|
|
5946
|
+
throw new SessionOomStoppedError(true);
|
|
5947
|
+
}
|
|
5948
|
+
if (!opts?.explicit && this.sessionOomStopped(name))
|
|
5949
|
+
throw new SessionOomStoppedError(false, readSessionOomStop(this.getInstanceDir(name)));
|
|
5658
5950
|
// One restart at a time per instance. Multiple sources can ask concurrently
|
|
5659
5951
|
// (MCP revival, /restart, pty_error, model failover); a second stop/start
|
|
5660
5952
|
// interleaved with the first tears down the window the first just created.
|
|
5661
5953
|
// Later callers join the in-flight restart instead — its opts win.
|
|
5662
5954
|
const inFlight = this.restartsInFlight.get(name);
|
|
5663
5955
|
if (inFlight) {
|
|
5956
|
+
// Do not let a late explicit caller clear a stop belonging to the
|
|
5957
|
+
// automatic flight it would join. That action must be retried afterwards.
|
|
5958
|
+
if (retry && this.sessionOomStopped(name))
|
|
5959
|
+
throw new SessionOomStoppedError(true);
|
|
5664
5960
|
this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
|
|
5665
5961
|
// #1753: a credential-profile switch joining it still needs its conversation carried in that restart's stop →
|
|
5666
5962
|
// start gap: handed to the restart while it has not reached the gap (and carries nothing else); otherwise not
|
|
@@ -5687,7 +5983,7 @@ export class FleetManager {
|
|
|
5687
5983
|
workingDirectory,
|
|
5688
5984
|
reason: "restart",
|
|
5689
5985
|
stage: "lifecycle",
|
|
5690
|
-
}, () => this.doRestartSingleInstance(name, opts, transition)))
|
|
5986
|
+
}, () => this.doRestartSingleInstance(name, opts, transition, retry)))
|
|
5691
5987
|
.finally(() => {
|
|
5692
5988
|
this.restartsInFlight.delete(name);
|
|
5693
5989
|
if (this.restartGaps.get(name) === gap)
|
|
@@ -5728,7 +6024,7 @@ export class FleetManager {
|
|
|
5728
6024
|
this.lifecycle.clearActivitySeedNow(name);
|
|
5729
6025
|
}
|
|
5730
6026
|
}
|
|
5731
|
-
async doRestartSingleInstance(name, opts, transition) {
|
|
6027
|
+
async doRestartSingleInstance(name, opts, transition, retry) {
|
|
5732
6028
|
if (this.configPath) {
|
|
5733
6029
|
this.loadConfig(this.configPath);
|
|
5734
6030
|
this.routing.rebuild(this.fleetConfig);
|
|
@@ -5746,7 +6042,7 @@ export class FleetManager {
|
|
|
5746
6042
|
if (!wasPaused)
|
|
5747
6043
|
this.restartsHoldingSlot.add(name);
|
|
5748
6044
|
try {
|
|
5749
|
-
await this.doRestartWithSlot(name, opts, transition, config, wasPaused);
|
|
6045
|
+
await this.doRestartWithSlot(name, opts, transition, config, wasPaused, retry);
|
|
5750
6046
|
// An explicit restart of a paused instance is an operator wake (§1.6):
|
|
5751
6047
|
// its success lifts the coordinator's park/backoff (the level stays).
|
|
5752
6048
|
// Automatic restarts never do.
|
|
@@ -5758,7 +6054,7 @@ export class FleetManager {
|
|
|
5758
6054
|
this.releaseWakeSlot(name, slot);
|
|
5759
6055
|
}
|
|
5760
6056
|
}
|
|
5761
|
-
async doRestartWithSlot(name, opts, transition, config, wasPaused) {
|
|
6057
|
+
async doRestartWithSlot(name, opts, transition, config, wasPaused, retry) {
|
|
5762
6058
|
const pausedAt = wasPaused ? (this.lifecycle.getLastPausedAt(name) ?? readPausedAt(this.getInstanceDir(name))) : null;
|
|
5763
6059
|
const pauseReason = wasPaused ? readPauseReason(this.getInstanceDir(name)) : null;
|
|
5764
6060
|
// #1753: the directory the CLI being stopped was launched in — its conversation is under that directory's key.
|
|
@@ -5766,6 +6062,12 @@ export class FleetManager {
|
|
|
5766
6062
|
// Between the stop and the start: the old CLI's transcript is complete, the new one has not looked yet.
|
|
5767
6063
|
/** `launching`: the login the start will use — read only when a switch is to be carried. */
|
|
5768
6064
|
const betweenStopAndStart = async (launching) => {
|
|
6065
|
+
// Clearing is synchronous, after the old daemon is stopped and inside
|
|
6066
|
+
// this action's serialized transition; an unseen later stop is retained.
|
|
6067
|
+
if (retry)
|
|
6068
|
+
clearSessionOomStopForRetry(this.getInstanceDir(name), retry);
|
|
6069
|
+
else
|
|
6070
|
+
assertNoSessionOomStop(this.getInstanceDir(name));
|
|
5769
6071
|
// #1753: this restart's own carry, or one a caller joined it with before this point (restartSingleInstance).
|
|
5770
6072
|
const gap = this.restartGaps.get(name);
|
|
5771
6073
|
const carry = gap?.carry ?? opts?.carry;
|
|
@@ -6533,7 +6835,8 @@ export class FleetManager {
|
|
|
6533
6835
|
const [adapterId, userId] = k.split("\u0000");
|
|
6534
6836
|
accounts.set(k, { adapterId, userId, sessions: 0, lastSeen: null });
|
|
6535
6837
|
}
|
|
6536
|
-
|
|
6838
|
+
// #2348: every granted account is listed too, signed in or not.
|
|
6839
|
+
for (const key of [...Object.keys(cfg?.narrow ?? {}), ...Object.keys(cfg?.grants ?? {})]) {
|
|
6537
6840
|
const at = key.indexOf(":"), adapterId = key.slice(0, at), userId = key.slice(at + 1), k = `${adapterId}\u0000${userId}`;
|
|
6538
6841
|
if (at > 0 && !accounts.has(k))
|
|
6539
6842
|
accounts.set(k, { adapterId, userId, sessions: 0, lastSeen: null });
|
|
@@ -6542,10 +6845,153 @@ export class FleetManager {
|
|
|
6542
6845
|
const owner = { adapterId: a.adapterId, userId: a.userId };
|
|
6543
6846
|
const badge = this.guestBadge(owner);
|
|
6544
6847
|
// `reach`: before narrowing — what Settings offers to narrow to (#2104 review: never an agent it cannot reach).
|
|
6848
|
+
const grant = own(cfg?.grants, `${a.adapterId}:${a.userId}`);
|
|
6849
|
+
const sent = this.guestCodeSent.get(`${a.adapterId}:${a.userId}`);
|
|
6545
6850
|
return { ...a, name: badge.name, platform: badge.platform, scope: [...this.guestScope(owner)].sort(), reach: [...this.guestReach(owner)].sort(),
|
|
6546
|
-
devices: devices.get(k) ?? 0, narrow: own(cfg?.narrow, `${a.adapterId}:${a.userId}`) ?? null
|
|
6851
|
+
devices: devices.get(k) ?? 0, narrow: own(cfg?.narrow, `${a.adapterId}:${a.userId}`) ?? null,
|
|
6852
|
+
// #2348: the grant as fleet.yaml has it (null: none), and the last code an admin sent from Settings.
|
|
6853
|
+
grant: grant ? { ...(grant.name ? { name: grant.name } : {}), instances: [...grant.instances] } : null,
|
|
6854
|
+
...(sent ? { code: { at: sent.at, outcome: sent.outcome.kind, ...(sent.outcome.kind === "sent" ? { expiresAt: sent.at + sent.outcome.expiresInMs } : {}),
|
|
6855
|
+
...(sent.outcome.kind === "dm_failed" ? { reason: sent.outcome.reason, platform: sent.outcome.platform, ...(sent.outcome.startUrl ? { startUrl: sent.outcome.startUrl } : {}) } : {}) } } : {}) };
|
|
6547
6856
|
}).sort((x, y) => (y.lastSeen ?? -1) - (x.lastSeen ?? -1));
|
|
6548
|
-
|
|
6857
|
+
// #2348: `explicit` — grants are the scope (default deny); false: the derived scope of #1920, until grants are saved.
|
|
6858
|
+
return { enabled: cfg?.enabled === true, chat: cfg?.chat !== false, explicit: cfg?.grants !== undefined, instances: Object.keys(this.fleetConfig?.instances ?? {}).sort(), connections, guests,
|
|
6859
|
+
audit: this.guestAudit() };
|
|
6860
|
+
}
|
|
6861
|
+
/**
|
|
6862
|
+
* #2499 review: the guest audit an admin reads — the newest grant changes and admin-sent codes from the event log
|
|
6863
|
+
* (`fleet` rows: no agent's Log shows them). Through guestOverview only: Settings' admin gate, never over the public
|
|
6864
|
+
* link, never a guest's. The person's name as the card shows it; a session handle stays on the server (`by` → whether
|
|
6865
|
+
* a person saved it here or it was confirmed in General).
|
|
6866
|
+
*/
|
|
6867
|
+
guestAudit(limit = FleetManager.GUEST_AUDIT_SHOWN) {
|
|
6868
|
+
if (!this.eventLog)
|
|
6869
|
+
return [];
|
|
6870
|
+
// One read, newest first by row id: a total order (created_at is to the second), so the cap never drops a newer row.
|
|
6871
|
+
let rows = [];
|
|
6872
|
+
try {
|
|
6873
|
+
rows = this.eventLog.latestOfTypes("fleet", ["guest_grant_changed", "guest_code_sent"], limit);
|
|
6874
|
+
}
|
|
6875
|
+
catch {
|
|
6876
|
+
return [];
|
|
6877
|
+
}
|
|
6878
|
+
const names = (list) => (Array.isArray(list) ? list.filter((n) => typeof n === "string") : []);
|
|
6879
|
+
return rows.flatMap((r) => {
|
|
6880
|
+
const p = r.payload ?? {}, account = typeof p.account === "string" ? p.account : "";
|
|
6881
|
+
const at = account.indexOf(":");
|
|
6882
|
+
if (at <= 0)
|
|
6883
|
+
return [];
|
|
6884
|
+
const name = this.webPerson({ adapterId: account.slice(0, at), userId: account.slice(at + 1) }).name;
|
|
6885
|
+
const when = Date.parse(r.created_at.includes("T") ? r.created_at : `${r.created_at.replace(" ", "T")}Z`);
|
|
6886
|
+
return r.event_type === "guest_grant_changed"
|
|
6887
|
+
? [{ kind: "grant", at: when, account, name, via: p.via === "confirmed" ? "confirmed" : "direct", added: names(p.added), removed: names(p.removed),
|
|
6888
|
+
endedSessions: typeof p.endedSessions === "number" ? p.endedSessions : 0 }]
|
|
6889
|
+
: [{ kind: "code", at: when, account, name, outcome: typeof p.outcome === "string" ? p.outcome : "unknown" }];
|
|
6890
|
+
});
|
|
6891
|
+
}
|
|
6892
|
+
static GUEST_AUDIT_SHOWN = 20;
|
|
6893
|
+
/**
|
|
6894
|
+
* #2348: Settings → Guests' picker — the people AgEnD has seen in chat (seen_people, from human receipts: never a bot,
|
|
6895
|
+
* never the web), without fleet admins (they sign in as admins) or accounts of a connection the fleet no longer has.
|
|
6896
|
+
* Admin only (settings-api.ts), never over the public link.
|
|
6897
|
+
*/
|
|
6898
|
+
guestPeople() {
|
|
6899
|
+
const seen = this.deliveryOutbox?.isOpen ? this.deliveryOutbox.seenPeople() : [];
|
|
6900
|
+
const people = seen.filter((p) => this.worlds.has(p.adapterId) && !this.isFleetAdmin(p.userId, p.adapterId)).map((p) => {
|
|
6901
|
+
const owner = { adapterId: p.adapterId, userId: p.userId };
|
|
6902
|
+
return { adapterId: p.adapterId, userId: p.userId, platform: this.worlds.get(p.adapterId)?.adapter.type ?? p.adapterId, username: p.username,
|
|
6903
|
+
name: this.webPerson(owner).name, lastSeen: p.lastSeen, instances: p.instances.map((i) => i.name) };
|
|
6904
|
+
});
|
|
6905
|
+
return { people };
|
|
6906
|
+
}
|
|
6907
|
+
/**
|
|
6908
|
+
* #2348: why this account may not be given a grant, or null. Only someone AgEnD has seen in chat, on a connection the
|
|
6909
|
+
* fleet runs, who is not a fleet admin of it (an admin is never a guest).
|
|
6910
|
+
*/
|
|
6911
|
+
guestGrantRefusal(adapterId, userId) {
|
|
6912
|
+
if (!this.worlds.has(adapterId) && !this.fleetConfig?.channels?.some((c) => (c.id ?? c.type) === adapterId))
|
|
6913
|
+
return "no such connection in this fleet";
|
|
6914
|
+
if (this.isFleetAdmin(userId, adapterId))
|
|
6915
|
+
return "a fleet admin signs in as an admin, not as a guest";
|
|
6916
|
+
if (!this.deliveryOutbox?.isOpen || !this.deliveryOutbox.isSeenPerson(adapterId, userId))
|
|
6917
|
+
return "only someone AgEnD has seen in chat can be given guest access";
|
|
6918
|
+
return null;
|
|
6919
|
+
}
|
|
6920
|
+
/**
|
|
6921
|
+
* #2348 delta §5: a grant changed with a save — one event-log entry per person, shown in Settings → Guests' Recent
|
|
6922
|
+
* changes (guestOverview: admin-only, never over the public link or to a guest): who saved it (an admin's
|
|
6923
|
+
* session handle, the header token, or null when replayed for a confirmation), whether it applied at once or was
|
|
6924
|
+
* confirmed in General, the agents added and taken away, and the sessions a removal ended. Never message content.
|
|
6925
|
+
*/
|
|
6926
|
+
noteGuestGrantChange(change) {
|
|
6927
|
+
this.logger.info({ account: change.account, via: change.via, added: change.added.length, removed: change.removed.length }, "Guest access changed in Settings");
|
|
6928
|
+
try {
|
|
6929
|
+
this.eventLog?.insert("fleet", "guest_grant_changed", { ...change });
|
|
6930
|
+
}
|
|
6931
|
+
catch { /* the log line stands */ }
|
|
6932
|
+
}
|
|
6933
|
+
/** #2348: an admin's Send sign-in code, per admin (the caller): at most this many in ADMIN_CODE_WINDOW_MS. */
|
|
6934
|
+
static ADMIN_CODE_MAX = 10;
|
|
6935
|
+
static ADMIN_CODE_WINDOW_MS = 10 * 60_000;
|
|
6936
|
+
adminCodeSends = new Map();
|
|
6937
|
+
/** #2348: the last admin-sent code per account, for the Settings card ("Code sent 18:42 · expires 18:52", or why not). */
|
|
6938
|
+
guestCodeSent = new Map();
|
|
6939
|
+
/**
|
|
6940
|
+
* #2348: Settings → Guests' "Send sign-in code" — a guest code to an account an admin granted access, by DM only (no
|
|
6941
|
+
* chat to answer in, nothing posted anywhere). The same issuance as the person's own /web (one code per account, the
|
|
6942
|
+
* code store's caps and breaker, the public link's audience when it is open) and the same once-a-minute limit per
|
|
6943
|
+
* account, plus a per-admin cap. Refused without guest sign-in, for a fleet admin, or with nothing granted. Audited.
|
|
6944
|
+
*/
|
|
6945
|
+
async sendGuestCodeByAdmin(owner, by,
|
|
6946
|
+
/** #2365 review: the initiating admin is still signed in and still an admin (settings-api re-checks the session). */
|
|
6947
|
+
current = () => true) {
|
|
6948
|
+
const guests = this.fleetConfig?.web?.guests;
|
|
6949
|
+
if (guests?.enabled !== true)
|
|
6950
|
+
return { status: 409, body: { ok: false, outcome: "guests_off", error: "Guest sign-in is off" } };
|
|
6951
|
+
if (this.isFleetAdmin(owner.userId, owner.adapterId))
|
|
6952
|
+
return { status: 409, body: { ok: false, outcome: "admin", error: "A fleet admin signs in as an admin" } };
|
|
6953
|
+
if (!this.guestScope(owner).size)
|
|
6954
|
+
return { status: 409, body: { ok: false, outcome: "nothing", error: "Nothing is granted to this person yet: save their access first" } };
|
|
6955
|
+
const token = this.webToken;
|
|
6956
|
+
const adapter = this.adapters.get(owner.adapterId);
|
|
6957
|
+
if (this.shuttingDown || !token || !adapter)
|
|
6958
|
+
return { status: 503, body: { ok: false, outcome: "starting", error: "AgEnD is starting or this connection is not running" } };
|
|
6959
|
+
const key = `${owner.adapterId}\u0000${owner.userId}`, asked = this.guestCodeAskedAt.get(key);
|
|
6960
|
+
const now = performance.now();
|
|
6961
|
+
if (this.guestCodeInFlight.has(key) || (asked !== undefined && now - asked < 60_000)) {
|
|
6962
|
+
return { status: 429, body: { ok: false, outcome: "wait", retryAfterMs: asked !== undefined ? Math.max(0, 60_000 - (now - asked)) : 60_000 } };
|
|
6963
|
+
}
|
|
6964
|
+
const mine = (this.adminCodeSends.get(by) ?? []).filter((at) => now - at < FleetManager.ADMIN_CODE_WINDOW_MS);
|
|
6965
|
+
if (mine.length >= FleetManager.ADMIN_CODE_MAX)
|
|
6966
|
+
return { status: 429, body: { ok: false, outcome: "admin_limit", retryAfterMs: FleetManager.ADMIN_CODE_WINDOW_MS - (now - mine[0]) } };
|
|
6967
|
+
this.adminCodeSends.set(by, [...mine, now]);
|
|
6968
|
+
while (this.adminCodeSends.size > 256)
|
|
6969
|
+
this.adminCodeSends.delete(this.adminCodeSends.keys().next().value);
|
|
6970
|
+
let outcome = { kind: "busy" };
|
|
6971
|
+
this.guestCodeInFlight.add(key);
|
|
6972
|
+
try {
|
|
6973
|
+
await this.issueGuestCode({ replyHere: async () => undefined }, owner, key, adapter, token, (o) => { outcome = o; }, current);
|
|
6974
|
+
}
|
|
6975
|
+
finally {
|
|
6976
|
+
this.guestCodeInFlight.delete(key);
|
|
6977
|
+
}
|
|
6978
|
+
const done = outcome;
|
|
6979
|
+
// #2365 review: the admin's session ended (or stopped being an admin's) while the code was on its way: nothing was
|
|
6980
|
+
// sent, or what was made is withdrawn — the answer says so, as to anyone no longer signed in.
|
|
6981
|
+
if (done.kind === "revoked" && !current())
|
|
6982
|
+
return { status: 401, body: { ok: false, outcome: "revoked", error: "no longer authorized" } };
|
|
6983
|
+
this.guestCodeSent.set(`${owner.adapterId}:${owner.userId}`, { at: Date.now(), outcome: done });
|
|
6984
|
+
while (this.guestCodeSent.size > 512)
|
|
6985
|
+
this.guestCodeSent.delete(this.guestCodeSent.keys().next().value);
|
|
6986
|
+
// Audited: who asked, for whom, what happened (never the code).
|
|
6987
|
+
this.logger.info({ by, adapterId: owner.adapterId, outcome: done.kind }, "Guest sign-in code sent by an admin from Settings");
|
|
6988
|
+
try {
|
|
6989
|
+
this.eventLog?.insert("fleet", "guest_code_sent", { by, account: `${owner.adapterId}:${owner.userId}`, outcome: done.kind, instances: this.guestScope(owner).size });
|
|
6990
|
+
}
|
|
6991
|
+
catch { /* the log line stands */ }
|
|
6992
|
+
const status = done.kind === "sent" ? 200 : done.kind === "dm_failed" ? 502 : 409;
|
|
6993
|
+
return { status, body: { ok: done.kind === "sent", outcome: done.kind, ...(done.kind === "sent" ? { expiresAt: Date.now() + done.expiresInMs, publicLink: done.publicLink } : {}),
|
|
6994
|
+
...(done.kind === "dm_failed" ? { platform: done.platform, reason: done.reason, ...(done.startUrl ? { startUrl: done.startUrl } : {}) } : {}) } };
|
|
6549
6995
|
}
|
|
6550
6996
|
/** #1920 phase 3 (the app shell): a guest's chip — its web name, and its connection's platform (discord, telegram). */
|
|
6551
6997
|
guestBadge(owner) {
|
|
@@ -6614,24 +7060,35 @@ export class FleetManager {
|
|
|
6614
7060
|
return true;
|
|
6615
7061
|
}
|
|
6616
7062
|
/** guestWebCommand's issuance, one per account at a time; a revoke made meanwhile (either kind) ends it unissued. */
|
|
6617
|
-
async issueGuestCode(ask, owner, key, adapter, token
|
|
7063
|
+
async issueGuestCode(ask, owner, key, adapter, token,
|
|
7064
|
+
/** #2348: how it ended, for a caller with no chat to answer in (Settings → Guests' Send sign-in code). */
|
|
7065
|
+
report = () => { },
|
|
7066
|
+
/**
|
|
7067
|
+
* #2365 review: the one who asked may still ask — for an admin's Send sign-in code, that admin's session as it is
|
|
7068
|
+
* now. Asked with the revocation checks, after every wait: false ends the issuance, and withdraws its code if one was made.
|
|
7069
|
+
*/
|
|
7070
|
+
stillAllowed = () => true) {
|
|
6618
7071
|
const say = (text) => ask.replyHere(text).catch(() => undefined);
|
|
6619
7072
|
const revokes = this.guestRevokes, accountRevokes = this.guestAccountRevokes.get(key) ?? 0;
|
|
6620
|
-
const unrevoked = () => this.guestRevokes === revokes && (this.guestAccountRevokes.get(key) ?? 0) === accountRevokes;
|
|
7073
|
+
const unrevoked = () => this.guestRevokes === revokes && (this.guestAccountRevokes.get(key) ?? 0) === accountRevokes && stillAllowed();
|
|
6621
7074
|
// §3.2: its rooms are read at sign-in (bounded; never longer than the reads' own deadline).
|
|
6622
7075
|
await withinBudget(this.refreshGuestRooms(owner, { mode: "force" }), performance.now() + 6_000).catch(() => undefined);
|
|
6623
|
-
if (!unrevoked())
|
|
6624
|
-
|
|
7076
|
+
if (!unrevoked()) {
|
|
7077
|
+
report({ kind: "revoked" });
|
|
7078
|
+
return;
|
|
7079
|
+
} // revoked while its rooms were read: no code
|
|
6625
7080
|
const scope = this.guestScope(owner);
|
|
6626
7081
|
if (!scope.size) {
|
|
6627
7082
|
const open = this.worlds.get(owner.adapterId)?.accessManager.getMode() === "open";
|
|
6628
7083
|
await say(t(open ? "web.guest_not_set_up" : "web.guest_nothing"));
|
|
7084
|
+
report({ kind: "nothing" });
|
|
6629
7085
|
return;
|
|
6630
7086
|
}
|
|
6631
7087
|
this.initializeWebSessions();
|
|
6632
7088
|
const codes = this.webGuestLoginCodes;
|
|
6633
7089
|
if (codes.pausedForMs() > 0) {
|
|
6634
7090
|
await say(t("web.guest_paused"));
|
|
7091
|
+
report({ kind: "paused" });
|
|
6635
7092
|
return;
|
|
6636
7093
|
}
|
|
6637
7094
|
// §5.5 (step 4b): with the public link open, the code works through it too (an admin opened it; a guest is still a
|
|
@@ -6642,6 +7099,7 @@ export class FleetManager {
|
|
|
6642
7099
|
recipient: { adapterId: owner.adapterId, userId: owner.userId, ...(ask.username ? { username: ask.username } : {}) } });
|
|
6643
7100
|
if (!login) {
|
|
6644
7101
|
await say(t("web.guest_busy"));
|
|
7102
|
+
report({ kind: "busy" });
|
|
6645
7103
|
return;
|
|
6646
7104
|
}
|
|
6647
7105
|
this.guestCodeAskedAt.delete(key);
|
|
@@ -6676,13 +7134,17 @@ export class FleetManager {
|
|
|
6676
7134
|
throw new Error("guest no longer admitted");
|
|
6677
7135
|
await say(t("web.guest_sent"));
|
|
6678
7136
|
}
|
|
7137
|
+
report({ kind: "sent", expiresInMs: LOGIN_CODE_TTL_MS, publicLink: !!open });
|
|
6679
7138
|
}
|
|
6680
7139
|
catch (err) {
|
|
6681
7140
|
codes.revokeIfCurrent(login.issuanceId); // this flow's code only, never a later one's
|
|
6682
7141
|
this.logger.info({ err: err?.message, adapterId: owner.adapterId }, "Guest sign-in code was not delivered privately");
|
|
6683
|
-
if (!unrevoked())
|
|
6684
|
-
|
|
7142
|
+
if (!unrevoked()) {
|
|
7143
|
+
report({ kind: "revoked" });
|
|
7144
|
+
return;
|
|
7145
|
+
} // revoked meanwhile: the revoke's own answer said so
|
|
6685
7146
|
const start = adapter.type === "telegram" ? this.tgStartWebUrl(owner.adapterId) : null;
|
|
7147
|
+
report({ kind: "dm_failed", platform: adapter.type, reason: String(err?.message ?? err).slice(0, 200), ...(start ? { startUrl: start } : {}) });
|
|
6686
7148
|
// #1742 for guests (step 4b-3): its START button (a bare /start, or the link's /start web) then asks again, from
|
|
6687
7149
|
// the DM — once, within the window.
|
|
6688
7150
|
if (adapter.type === "telegram")
|
|
@@ -7229,6 +7691,7 @@ export class FleetManager {
|
|
|
7229
7691
|
const startupStartedAt = Date.now();
|
|
7230
7692
|
FleetManager.signalTarget = this;
|
|
7231
7693
|
this.startupComplete = false;
|
|
7694
|
+
this.workingRecovery?.reset();
|
|
7232
7695
|
this.startupDashboard = null;
|
|
7233
7696
|
// Cleared here, not at the end of doStopAll: a stop has an async tail, and
|
|
7234
7697
|
// anything arriving during it is still part of the stop.
|
|
@@ -7807,12 +8270,17 @@ export class FleetManager {
|
|
|
7807
8270
|
// health server — so `systemctl start` returned success while the fleet was
|
|
7808
8271
|
// still deaf: no path existed for a user message to arrive.
|
|
7809
8272
|
sdNotify("READY=1");
|
|
8273
|
+
this.workingRecovery?.ready();
|
|
7810
8274
|
// #2481: this version runs — the CLI's start-failure banner stops, and this is the version to go back to later.
|
|
7811
8275
|
try {
|
|
7812
8276
|
const { ownPackage, recordStarted } = await import("./start-failure.js");
|
|
7813
8277
|
const version = ownPackage().version;
|
|
7814
8278
|
if (version)
|
|
7815
8279
|
recordStarted(this.dataDir, version, process.pid, Date.now());
|
|
8280
|
+
// #2481: a start completed — the failed tree an automatic restore kept for a report can go.
|
|
8281
|
+
const prefix = ownPackage().prefix;
|
|
8282
|
+
if (prefix)
|
|
8283
|
+
void (await import("./mac-auto-restore.js")).pruneFailedTrees(prefix).catch(() => { });
|
|
7816
8284
|
}
|
|
7817
8285
|
catch (err) {
|
|
7818
8286
|
this.logger.debug({ err }, "fleet-started record not written");
|
|
@@ -9702,7 +10170,7 @@ export class FleetManager {
|
|
|
9702
10170
|
...generalReactions.meta,
|
|
9703
10171
|
...extraMeta,
|
|
9704
10172
|
},
|
|
9705
|
-
}, msg.humanOrigin, msg.humanOriginalBody ?? msg.text);
|
|
10173
|
+
}, msg.humanOrigin, msg.humanOriginalBody ?? msg.text, this.startRetryHoldFor(generalInstance, msg));
|
|
9706
10174
|
generalReactions.consume();
|
|
9707
10175
|
this.afterUserInboundDelivered(generalInstance, msg);
|
|
9708
10176
|
this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
|
|
@@ -9798,7 +10266,7 @@ export class FleetManager {
|
|
|
9798
10266
|
...reactions.meta,
|
|
9799
10267
|
...extraMeta,
|
|
9800
10268
|
},
|
|
9801
|
-
}, msg.humanOrigin, msg.humanOriginalBody ?? msg.text);
|
|
10269
|
+
}, msg.humanOrigin, msg.humanOriginalBody ?? msg.text, this.startRetryHoldFor(instanceName, msg));
|
|
9802
10270
|
// Only after the message actually went out. A failed delivery keeps the
|
|
9803
10271
|
// reactions queued for the retry / the next message.
|
|
9804
10272
|
reactions.consume();
|
|
@@ -10477,6 +10945,7 @@ export class FleetManager {
|
|
|
10477
10945
|
// overwritten with coordinates the target's adapter cannot address.
|
|
10478
10946
|
...(foreignWorld ? {} : { chat_id: reply_chat_id, thread_id: reply_thread_id }),
|
|
10479
10947
|
user: "scheduler",
|
|
10948
|
+
source: "schedule",
|
|
10480
10949
|
...(adapterId ? { adapter_id: adapterId } : {}),
|
|
10481
10950
|
},
|
|
10482
10951
|
}, { waitForIdle: true, ...(retryCurrent ? { stillCurrent: retryCurrent } : {}) });
|
|
@@ -14571,7 +15040,7 @@ export class FleetManager {
|
|
|
14571
15040
|
...(input.instance !== undefined ? { target: { adapterId: this.getInstanceAdapterId(input.instance) } } : {}),
|
|
14572
15041
|
isAdmin: (userId, adapterId) => this.adminGate(userId, adapterId) === "ok",
|
|
14573
15042
|
authorization: input.session ? this.sessionAuthorizationStore()?.get(input.session.handle) ?? null : null,
|
|
14574
|
-
policy: { ...webAuthorityPolicy(this.fleetConfig?.web), enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink }, epoch: state.epoch, now: Date.now(),
|
|
15043
|
+
policy: { ...webAuthorityPolicy(this.fleetConfig?.web), enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink, quickLink: state.quickLink }, epoch: state.epoch, now: Date.now(),
|
|
14575
15044
|
});
|
|
14576
15045
|
}
|
|
14577
15046
|
sessionAuthorizations = null;
|
|
@@ -14596,13 +15065,13 @@ export class FleetManager {
|
|
|
14596
15065
|
session, gateway, peer, tokenEpoch: this.webToken ? tokenEpoch(this.webToken) : null,
|
|
14597
15066
|
isAdmin: (userId, adapterId) => this.adminGate(userId, adapterId) === "ok",
|
|
14598
15067
|
authorization: { handle: session.handle, owner: session.owner, authorizedAt: now, until: now + policy.maxSessionMs, epoch: state.epoch },
|
|
14599
|
-
policy: { ...policy, enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink }, epoch: state.epoch, now,
|
|
15068
|
+
policy: { ...policy, enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink, quickLink: state.quickLink }, epoch: state.epoch, now,
|
|
14600
15069
|
});
|
|
14601
15070
|
if (check.refusal)
|
|
14602
15071
|
return { ok: false, refusal: check.refusal };
|
|
14603
15072
|
const live = new Set(this.webSessions?.list(undefined, () => true).map(r => r.handle) ?? []);
|
|
14604
15073
|
store.prune(a => a.epoch === state.epoch && now <= Math.min(a.until, a.authorizedAt + policy.maxSessionMs) && live.has(a.handle));
|
|
14605
|
-
return { ok: true, ...store.authorize(session.handle, session.owner, now, state.epoch, now + policy.maxSessionMs) };
|
|
15074
|
+
return { ok: true, ...store.authorize(session.handle, session.owner, now, state.epoch, now + policy.maxSessionMs, peer) };
|
|
14606
15075
|
}
|
|
14607
15076
|
authorityStateStore = null;
|
|
14608
15077
|
/**
|
|
@@ -14611,13 +15080,24 @@ export class FleetManager {
|
|
|
14611
15080
|
*/
|
|
14612
15081
|
webAuthorityState() {
|
|
14613
15082
|
if (!this.dataDir)
|
|
14614
|
-
return { enabled: false, loopbackOnly: true, namedLink: false, epoch: 0 };
|
|
15083
|
+
return { enabled: false, loopbackOnly: true, namedLink: false, quickLink: false, epoch: 0 };
|
|
14615
15084
|
this.authorityStateStore ??= new AuthorityStateStore(join(this.dataDir, "web-authority.json"), this.logger, undefined, undefined, () => { this.ensureDeliveryOutbox?.(); return this.deliveryOutbox?.maxWebAuthorityEpoch() ?? 0; });
|
|
14616
15085
|
const state = this.authorityStateStore.current(this.fleetConfig?.web?.verified_authority);
|
|
14617
|
-
// #2453 (§0.2):
|
|
14618
|
-
// good: their authorizations go (a later "on" can never bring one back without its account's Confirm).
|
|
14619
|
-
|
|
14620
|
-
|
|
15086
|
+
// #2453 (§0.2) / §0.3: a link's option seen going off — the authority of every session over that link ends now, for
|
|
15087
|
+
// good: their authorizations go (a later "on" can never bring one back without its account's Confirm). A record from
|
|
15088
|
+
// before §0.3 has no peer: over the gateway, only a named-link session could have been authorized then.
|
|
15089
|
+
if (state.namedLinkClosed || state.quickLinkClosed) {
|
|
15090
|
+
const ended = (a) => {
|
|
15091
|
+
const session = this.webSessions?.findByHandle(a.handle);
|
|
15092
|
+
if (!session)
|
|
15093
|
+
return true;
|
|
15094
|
+
if (session.surface === "local")
|
|
15095
|
+
return false;
|
|
15096
|
+
const peer = a.peer ?? "named_link";
|
|
15097
|
+
return (!!state.namedLinkClosed && peer === "named_link") || (!!state.quickLinkClosed && peer === "quick_link");
|
|
15098
|
+
};
|
|
15099
|
+
this.sessionAuthorizationStore()?.prune(a => !ended(a));
|
|
15100
|
+
}
|
|
14621
15101
|
return state;
|
|
14622
15102
|
}
|
|
14623
15103
|
/**
|
|
@@ -14650,7 +15130,8 @@ export class FleetManager {
|
|
|
14650
15130
|
.filter(a => a.epoch === state.epoch && sessions.has(a.handle) && now <= Math.min(a.until, a.authorizedAt + maxMs))
|
|
14651
15131
|
.map(a => ({ handle: a.handle, label: sessions.get(a.handle).label, account: `${a.owner.adapterId}:${a.owner.userId}`, authorizedAt: a.authorizedAt,
|
|
14652
15132
|
until: Math.min(a.until, a.authorizedAt + maxMs, sessions.get(a.handle).absoluteExpiry),
|
|
14653
|
-
|
|
15133
|
+
// Over a link: the temporary one (§0.3), else the named one (a record from before §0.3 has no peer).
|
|
15134
|
+
...(sessions.get(a.handle).surface === "gateway" ? (a.peer === "quick_link" ? { quick: true } : { named: true }) : {}) })) : [];
|
|
14654
15135
|
const host = this.fleetConfig?.web?.public_link?.hostname;
|
|
14655
15136
|
return { state, configured, hours: VERIFIED_SESSION_HOURS, lanHosts, authorized, namedHost: typeof host === "string" && host ? host : null };
|
|
14656
15137
|
}
|
|
@@ -14665,9 +15146,17 @@ export class FleetManager {
|
|
|
14665
15146
|
const adapterOf = (adapterId) => this.adapters.get(adapterId) ?? this.worlds.get(adapterId)?.adapter;
|
|
14666
15147
|
// The host's clock, with its zone: the person may read the DM on a phone in another one.
|
|
14667
15148
|
const when = (ms) => new Date(ms).toLocaleString(undefined, { timeZoneName: "short" });
|
|
15149
|
+
const quickHost = () => { try {
|
|
15150
|
+
const u = this.publicWebLink?.openSignin()?.url;
|
|
15151
|
+
return u ? new URL(u).host : "?";
|
|
15152
|
+
}
|
|
15153
|
+
catch {
|
|
15154
|
+
return "?";
|
|
15155
|
+
} };
|
|
14668
15156
|
const where = (peer) => peer === "named_link"
|
|
14669
15157
|
? t("web_auth.peer_named_link", String(this.fleetConfig?.web?.public_link?.hostname ?? "?")) // §0.2: the internet
|
|
14670
|
-
:
|
|
15158
|
+
: peer === "quick_link" ? t("web_auth.peer_quick_link", quickHost()) // §0.3: the internet
|
|
15159
|
+
: t(peer === "loopback" ? "web_auth.peer_loopback" : "web_auth.peer_lan");
|
|
14671
15160
|
return this.authorizationRequests = new SessionAuthorizationRequests({
|
|
14672
15161
|
now: () => performance.now(), wallNow: () => Date.now(),
|
|
14673
15162
|
maxSessionMs: () => webAuthorityPolicy(this.fleetConfig?.web).maxSessionMs,
|
|
@@ -14701,8 +15190,8 @@ export class FleetManager {
|
|
|
14701
15190
|
const record = this.webSessions?.findByHandle(handle);
|
|
14702
15191
|
if (!record)
|
|
14703
15192
|
return { ok: false, refusal: "session_gone" };
|
|
14704
|
-
// §0.2: a
|
|
14705
|
-
const r = this.authorizeWebSession(record, peer, peer === "named_link");
|
|
15193
|
+
// §0.2 / §0.3: a session over a link is judged as the gateway request it is (its exposure still the current one).
|
|
15194
|
+
const r = this.authorizeWebSession(record, peer, peer === "named_link" || peer === "quick_link");
|
|
14706
15195
|
// What the card says: the end of the authorization, or the session's own end when that comes first.
|
|
14707
15196
|
return r.ok ? { ok: true, until: Math.min(r.authorization.until, record.absoluteExpiry) } : { ok: false, refusal: r.refusal };
|
|
14708
15197
|
},
|
|
@@ -14748,7 +15237,7 @@ export class FleetManager {
|
|
|
14748
15237
|
session: s, gateway: input.gateway, peer: input.peer, tokenEpoch: this.webToken ? tokenEpoch(this.webToken) : null,
|
|
14749
15238
|
isAdmin: (userId, adapterId) => this.adminGate(userId, adapterId) === "ok",
|
|
14750
15239
|
authorization: s?.owner ? { handle: s.handle, owner: s.owner, authorizedAt: now, until: now + policy.maxSessionMs, epoch: state.epoch } : null,
|
|
14751
|
-
policy: { ...policy, enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink }, epoch: state.epoch, now,
|
|
15240
|
+
policy: { ...policy, enabled: state.enabled, loopbackOnly: state.loopbackOnly, namedLink: state.namedLink, quickLink: state.quickLink }, epoch: state.epoch, now,
|
|
14752
15241
|
});
|
|
14753
15242
|
return r.refusal;
|
|
14754
15243
|
}
|
|
@@ -15021,14 +15510,42 @@ export class FleetManager {
|
|
|
15021
15510
|
/** A clean exit is intentional from the CLI's perspective, but often not from
|
|
15022
15511
|
* the operator's. Keep the instance notice passive and put the action in the
|
|
15023
15512
|
* same-world General topic where an administrator can make the choice. */
|
|
15024
|
-
async
|
|
15025
|
-
this.
|
|
15513
|
+
async notifySessionOomStop(instanceName, unclear = false) {
|
|
15514
|
+
await this.notifyNormalExit(instanceName, unclear ? "unclear" : true);
|
|
15515
|
+
}
|
|
15516
|
+
async notifyNormalExit(instanceName, oom = false) {
|
|
15517
|
+
const daemon = oom === "unclear" ? this.daemons.get(instanceName) : undefined;
|
|
15518
|
+
const owner = daemon ? { ...daemon.getInteractionSnapshot().owner } : undefined;
|
|
15519
|
+
const current = daemon && owner ? () => this.daemons.get(instanceName) === daemon
|
|
15520
|
+
&& daemon.isUnclearOomStopped() && sameInteractionOwner(owner, daemon.getInteractionSnapshot().owner) : undefined;
|
|
15521
|
+
if (oom === "unclear" && !current?.())
|
|
15522
|
+
return;
|
|
15523
|
+
const state = oom ? readSessionOomStop(this.getInstanceDir(instanceName)) : null;
|
|
15524
|
+
if (oom === "unclear" && state?.blocked)
|
|
15525
|
+
return;
|
|
15526
|
+
const proof = state?.blocked ? state.record?.evidence ?? state.evidence : undefined;
|
|
15527
|
+
if (oom === true && !proof)
|
|
15528
|
+
return;
|
|
15529
|
+
const sessionOomRetry = oom ? captureSessionOomRetry(this.getInstanceDir(instanceName)) : undefined;
|
|
15530
|
+
if (oom && [...this.pendingNonceButtons.values()].some(p => p.prefix === OOM_RETRY_CALLBACK_PREFIX
|
|
15531
|
+
&& p.instanceName === instanceName && p.sessionOomRetry?.revision === sessionOomRetry?.revision
|
|
15532
|
+
&& (oom !== "unclear" || p.confirmationCurrent?.() === true)))
|
|
15533
|
+
return;
|
|
15534
|
+
const prefix = oom ? OOM_RETRY_CALLBACK_PREFIX : EXIT_RESTART_CALLBACK_PREFIX;
|
|
15535
|
+
const message = oom === "unclear" ? t("inst.oom_unclear_choice", instanceName) : oom && proof
|
|
15536
|
+
? t("inst.session_oom_choice", instanceName, describeSelfOom(proof))
|
|
15537
|
+
: t("exit.general_notice", placeRef({ kind: "instance", name: instanceName }, instanceName));
|
|
15538
|
+
const choices = oom
|
|
15539
|
+
? [{ action: "resume", label: t("inst.session_oom_resume") }, { action: "fresh", label: t("inst.session_oom_fresh") }]
|
|
15540
|
+
: [{ action: "restart", label: t("exit.restart") }, { action: "ignore", label: t("exit.ignore") }];
|
|
15541
|
+
const extra = sessionOomRetry ? { sessionOomRetry, ...(current ? { confirmationCurrent: current } : {}) } : undefined;
|
|
15542
|
+
this.notifyInstanceTopic(instanceName, (oom ? message : t("exit.instance_notice", instanceName)) + placeSuffix({ kind: "general" }, "General"));
|
|
15026
15543
|
const web = this.webOnlyPromptPlace();
|
|
15027
15544
|
if (web) {
|
|
15028
15545
|
await this.postNonceButtonPrompt({
|
|
15029
|
-
prefix
|
|
15030
|
-
message
|
|
15031
|
-
choices
|
|
15546
|
+
prefix, alertType: "exit_restart", instanceName, ...web,
|
|
15547
|
+
message, extra,
|
|
15548
|
+
choices,
|
|
15032
15549
|
expiredText: t("exit.expired", instanceName),
|
|
15033
15550
|
});
|
|
15034
15551
|
return;
|
|
@@ -15049,37 +15566,36 @@ export class FleetManager {
|
|
|
15049
15566
|
return;
|
|
15050
15567
|
}
|
|
15051
15568
|
await this.postNonceButtonPrompt({
|
|
15052
|
-
prefix
|
|
15569
|
+
prefix,
|
|
15053
15570
|
alertType: "exit_restart",
|
|
15054
15571
|
instanceName,
|
|
15055
15572
|
adapter,
|
|
15056
15573
|
adapterId,
|
|
15057
15574
|
chatId,
|
|
15058
15575
|
threadId,
|
|
15059
|
-
message
|
|
15060
|
-
choices
|
|
15061
|
-
{ action: "restart", label: t("exit.restart") },
|
|
15062
|
-
{ action: "ignore", label: t("exit.ignore") },
|
|
15063
|
-
],
|
|
15576
|
+
message, extra,
|
|
15577
|
+
choices,
|
|
15064
15578
|
expiredText: t("exit.expired", instanceName),
|
|
15065
15579
|
});
|
|
15066
15580
|
}
|
|
15067
15581
|
/** Consume a clean-exit Restart/Ignore button exactly once. */
|
|
15068
15582
|
async handleExitRestartPrompt(data, callbackAdapterId, receivingAdapter) {
|
|
15069
|
-
const
|
|
15583
|
+
const oom = data.callbackData.startsWith(OOM_RETRY_CALLBACK_PREFIX);
|
|
15584
|
+
const claimed = this.consumeNonceCallback(oom ? OOM_RETRY_CALLBACK_PREFIX : EXIT_RESTART_CALLBACK_PREFIX, oom ? /^oom-retry:([0-9a-f]+):(resume|fresh)$/ : /^exit-restart:([0-9a-f]+):(restart|ignore)$/, data, callbackAdapterId, receivingAdapter);
|
|
15070
15585
|
if (claimed === null)
|
|
15071
15586
|
return false;
|
|
15072
15587
|
if (claimed === "consumed")
|
|
15073
15588
|
return true;
|
|
15074
15589
|
const { entry: pending, action } = claimed;
|
|
15075
|
-
this.eventLog?.insert(pending.instanceName, "normal_exit_action", { action, userId: data.userId });
|
|
15590
|
+
this.eventLog?.insert(pending.instanceName, oom ? "session_oom_action" : "normal_exit_action", { action, userId: data.userId });
|
|
15076
15591
|
if (action === "ignore") {
|
|
15077
15592
|
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("exit.ignored", pending.instanceName));
|
|
15078
15593
|
return true;
|
|
15079
15594
|
}
|
|
15080
15595
|
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("exit.restarting", pending.instanceName));
|
|
15081
15596
|
try {
|
|
15082
|
-
await this.restartSingleInstance(pending.instanceName
|
|
15597
|
+
await this.restartSingleInstance(pending.instanceName, { explicit: true, ...(oom ? { freshStart: action === "fresh", sessionOomRetry: pending.sessionOomRetry,
|
|
15598
|
+
...(pending.confirmationCurrent ? { recoveryCurrent: pending.confirmationCurrent } : {}) } : {}) });
|
|
15083
15599
|
await pending.adapter.editMessage(pending.chatId, pending.messageId ?? data.messageId, t("exit.restarted", pending.instanceName), pending.threadId);
|
|
15084
15600
|
}
|
|
15085
15601
|
catch (err) {
|
|
@@ -16910,6 +17426,10 @@ export class FleetManager {
|
|
|
16910
17426
|
log: (level, message, extra) => this.logger[level](extra ?? {}, message),
|
|
16911
17427
|
});
|
|
16912
17428
|
db.board.onChange(({ rev, ids, projects, wakes }) => {
|
|
17429
|
+
// Refresh both old and new owners on reassignment; wakes are not an ownership index.
|
|
17430
|
+
if (ids.length)
|
|
17431
|
+
for (const name of this.daemons.keys())
|
|
17432
|
+
this.observeWorkingState(name, "tasks");
|
|
16913
17433
|
this.emitSseEvent("tasks", { rev, ids, projects });
|
|
16914
17434
|
waker.note(wakes);
|
|
16915
17435
|
// step 3: cards waiting for the person are Needs you items; #2355 review: a project's lead routes them too.
|
|
@@ -17008,7 +17528,7 @@ export class FleetManager {
|
|
|
17008
17528
|
}
|
|
17009
17529
|
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("hang.restarting", pending.instanceName));
|
|
17010
17530
|
try {
|
|
17011
|
-
await this.restartSingleInstance(pending.instanceName);
|
|
17531
|
+
await this.restartSingleInstance(pending.instanceName, { explicit: true });
|
|
17012
17532
|
await pending.adapter.editMessage(pending.chatId, pending.messageId ?? data.messageId, t("hang.restarted", pending.instanceName), pending.threadId);
|
|
17013
17533
|
}
|
|
17014
17534
|
catch (err) {
|
|
@@ -17544,7 +18064,12 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
17544
18064
|
const isPrimaryAdapter = !mentionWorld || mentionWorld.adapter === this.adapter;
|
|
17545
18065
|
const adapterBotUserId = mentionWorld?.botUserId ?? (isPrimaryAdapter ? this.botUserId : undefined);
|
|
17546
18066
|
const mentionTag = adapterBotUserId ? `<@${adapterBotUserId}>` : null;
|
|
17547
|
-
|
|
18067
|
+
// #2551: a native reply to one of THIS bot's messages is addressed to it, as a mention is (a Discord reply carries
|
|
18068
|
+
// its ping in the reference, not the text). One hop only: a reply to a message of ours that itself answered a bot
|
|
18069
|
+
// (or the sender) is a chain of bots answering each other — a mention still carries it on. A reply to our answer
|
|
18070
|
+
// to a person is the ordinary handoff, and is addressed.
|
|
18071
|
+
const repliesToUs = !!adapterBotUserId && msg.replyToAuthorId === adapterBotUserId && msg.replyToChain !== true;
|
|
18072
|
+
const isMentioned = (mentionTag && text.includes(mentionTag)) || repliesToUs;
|
|
17548
18073
|
if (!isMentioned) {
|
|
17549
18074
|
// Bare attachment (no @mention) — already saved above; just acknowledge.
|
|
17550
18075
|
if (saved) {
|
|
@@ -17780,7 +18305,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
17780
18305
|
content: fullText,
|
|
17781
18306
|
targetSession: instanceName,
|
|
17782
18307
|
meta,
|
|
17783
|
-
}, msg.humanOrigin, msg.humanOriginalBody ?? text);
|
|
18308
|
+
}, msg.humanOrigin, msg.humanOriginalBody ?? text, this.startRetryHoldFor(instanceName));
|
|
17784
18309
|
reactions.consume();
|
|
17785
18310
|
}
|
|
17786
18311
|
catch (err) {
|
|
@@ -19738,7 +20263,12 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
19738
20263
|
// also sees the channel's messages.
|
|
19739
20264
|
if (adapterId)
|
|
19740
20265
|
this.bindInstanceAdapter(instanceName, adapterId);
|
|
19741
|
-
|
|
20266
|
+
if (this.sessionOomStopped(instanceName)) {
|
|
20267
|
+
await this.restartSingleInstance(instanceName, { explicit: true });
|
|
20268
|
+
}
|
|
20269
|
+
else {
|
|
20270
|
+
await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
20271
|
+
}
|
|
19742
20272
|
this.reregisterClassicChannels();
|
|
19743
20273
|
// Auto-enable collab for Discord classic channels (TG uses @mention directly without collab mode)
|
|
19744
20274
|
if (guildId && !classicChannels.isCollab(channelId, adapterId)) {
|
|
@@ -19808,6 +20338,22 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
19808
20338
|
}
|
|
19809
20339
|
tmuxExecHelperTimer = null;
|
|
19810
20340
|
stopAllInFlight = null;
|
|
20341
|
+
/** Signal-handler continuation, after successful graceful shutdown only. */
|
|
20342
|
+
noteRequestedShutdown() {
|
|
20343
|
+
const progress = readUpdateProgress(this.dataDir)?.progress;
|
|
20344
|
+
const update = progress && updateProgressOperation(progress) === "update"
|
|
20345
|
+
&& ["installed", "stopping", "starting"].includes(progress.stage);
|
|
20346
|
+
this.recordWorkingRestartIntent(update ? "update" : "manual_restart");
|
|
20347
|
+
}
|
|
20348
|
+
recordWorkingRestartIntent(cause) {
|
|
20349
|
+
this.workingRecoveryCause = cause;
|
|
20350
|
+
try {
|
|
20351
|
+
this.deliveryOutbox?.workingRecovery.requested(this.managerBootId, cause);
|
|
20352
|
+
}
|
|
20353
|
+
catch {
|
|
20354
|
+
this.logger.warn("Could not record requested restart; next recovery will report an unknown interruption");
|
|
20355
|
+
}
|
|
20356
|
+
}
|
|
19811
20357
|
/** Stops release memory. Reassess between batches without waiting for recovery. */
|
|
19812
20358
|
async stopInstancesInBatches(entries, stop) {
|
|
19813
20359
|
let index = 0;
|
|
@@ -19825,15 +20371,25 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
19825
20371
|
}
|
|
19826
20372
|
}
|
|
19827
20373
|
async doStopAll() {
|
|
20374
|
+
// The scheduler closes before individual daemons stop. Preserve the owned
|
|
20375
|
+
// task/reply boundary while those stores are still available, not a later
|
|
20376
|
+
// shutdown snapshot whose unavailable sources cannot authorize recovery.
|
|
20377
|
+
for (const name of this.daemons.keys())
|
|
20378
|
+
this.observeWorkingState(name, "before_stop");
|
|
19828
20379
|
this.startupMemoryDeadline?.stop();
|
|
19829
20380
|
this.startupMemoryDeadline = null;
|
|
19830
20381
|
this.startupComplete = false;
|
|
20382
|
+
this.workingRecovery?.reset();
|
|
19831
20383
|
this.startupDashboard = null;
|
|
19832
20384
|
this.reloadPending = false;
|
|
19833
20385
|
// Before anything is stopped: everything that dies from here on dies
|
|
19834
20386
|
// because we asked it to. Set synchronously — doStopAll runs to its first
|
|
19835
20387
|
// await in the same tick as the signal handler, so no event can slip in.
|
|
19836
20388
|
this.shuttingDown = true;
|
|
20389
|
+
this.workingRecovery?.reset();
|
|
20390
|
+
if (this.workingRecoveryTimer)
|
|
20391
|
+
clearInterval(this.workingRecoveryTimer);
|
|
20392
|
+
this.workingRecoveryTimer = null;
|
|
19837
20393
|
if (this.updateProgressTimer) {
|
|
19838
20394
|
clearInterval(this.updateProgressTimer);
|
|
19839
20395
|
this.updateProgressTimer = null;
|
|
@@ -20513,6 +21069,27 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
20513
21069
|
fleetSignatureMismatchKeys() {
|
|
20514
21070
|
return this.fleetSignatureMismatch;
|
|
20515
21071
|
}
|
|
21072
|
+
/**
|
|
21073
|
+
* #2347 restart flow: saved changes that take effect only after AgEnD restarts — the startup-only keys fleet.yaml
|
|
21074
|
+
* now differs on, and the newest apply job whose fleet row can carry the restart (`job_id`, null when none can: a
|
|
21075
|
+
* hand edit, or a job past its retention). Null when nothing is pending, and when a restart could not clear it
|
|
21076
|
+
* (the signature mismatch the panel reports instead). The same test requestSettingsSelfRestart makes.
|
|
21077
|
+
*/
|
|
21078
|
+
settingsRestartPending() {
|
|
21079
|
+
if (this.appliedFleetLevel === null || this.fleetSignatureMismatch)
|
|
21080
|
+
return null;
|
|
21081
|
+
const next = this.nextFleetConfig();
|
|
21082
|
+
if (this.appliedFleetLevel === this.fleetLevelSignature(next))
|
|
21083
|
+
return null;
|
|
21084
|
+
let job = null;
|
|
21085
|
+
// A job whose restart was asked for already has its row in flight ("running"), so it is never offered again.
|
|
21086
|
+
for (const candidate of this.applyJobs.all()) {
|
|
21087
|
+
const row = candidate.targets.find(item => item.target === APPLY_FLEET_TARGET);
|
|
21088
|
+
if (row?.status === "restart-required" && (!job || candidate.startedAt >= job.startedAt))
|
|
21089
|
+
job = candidate;
|
|
21090
|
+
}
|
|
21091
|
+
return { keys: fleetLevelDifferences(this.startupFleetConfig, next), job_id: job ? job.id : null };
|
|
21092
|
+
}
|
|
20516
21093
|
/**
|
|
20517
21094
|
* Restart AgEnD itself on behalf of a Settings apply.
|
|
20518
21095
|
*
|
|
@@ -21763,6 +22340,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
21763
22340
|
this.logger.info("No instances to restart");
|
|
21764
22341
|
return;
|
|
21765
22342
|
}
|
|
22343
|
+
this.recordWorkingRestartIntent("manual_restart");
|
|
22344
|
+
this.workingRecovery?.reset();
|
|
21766
22345
|
this.logger.info(`Graceful restart: waiting for ${instanceNames.length} instances to idle...`);
|
|
21767
22346
|
const groupId = this.fleetConfig?.channel?.group_id;
|
|
21768
22347
|
const generalName = this.findGeneralInstance();
|
|
@@ -21863,7 +22442,16 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
21863
22442
|
this.startStatuslineWatcher(name);
|
|
21864
22443
|
}
|
|
21865
22444
|
}
|
|
22445
|
+
this.workingRecovery?.ready();
|
|
21866
22446
|
this.logger.info("Graceful restart complete");
|
|
22447
|
+
// The in-process interval keeps its label, but a later unrelated process
|
|
22448
|
+
// crash must not inherit this completed request as its shutdown intent.
|
|
22449
|
+
try {
|
|
22450
|
+
this.deliveryOutbox?.workingRecovery.clearRequested(this.managerBootId);
|
|
22451
|
+
}
|
|
22452
|
+
catch {
|
|
22453
|
+
this.logger.warn("Could not clear completed in-process restart intent");
|
|
22454
|
+
}
|
|
21867
22455
|
const configuredNames = this.configuredStartupInstanceNames(fleet, topicMode);
|
|
21868
22456
|
const total = configuredNames.length;
|
|
21869
22457
|
const started = configuredNames.filter(name => this.daemons.has(name)).length;
|
|
@@ -21918,6 +22506,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
21918
22506
|
}
|
|
21919
22507
|
const ipc = this.instanceIpcClients.get(name);
|
|
21920
22508
|
if (ipc?.connected) {
|
|
22509
|
+
daemon?.noteLegacyRestartReminder?.();
|
|
21921
22510
|
ipc.send({
|
|
21922
22511
|
type: "fleet_inbound",
|
|
21923
22512
|
content: msg,
|
|
@@ -22065,6 +22654,11 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
22065
22654
|
const guests = this.fleetConfig?.web?.guests;
|
|
22066
22655
|
if (!owner || guests?.enabled !== true)
|
|
22067
22656
|
return NO_GUEST_SCOPE;
|
|
22657
|
+
// #2348: explicit grants, once the key exists, are the whole scope — no narrowing on top (a grant is already exact).
|
|
22658
|
+
if (guests.grants !== undefined) {
|
|
22659
|
+
const granted = this.guestReach(owner);
|
|
22660
|
+
return granted.size ? granted : NO_GUEST_SCOPE;
|
|
22661
|
+
}
|
|
22068
22662
|
const scope = new Set(this.guestReach(owner));
|
|
22069
22663
|
const narrow = guests.narrow && Object.hasOwn(guests.narrow, `${owner.adapterId}:${owner.userId}`) ? guests.narrow[`${owner.adapterId}:${owner.userId}`] : undefined;
|
|
22070
22664
|
if (narrow) {
|
|
@@ -22085,6 +22679,20 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
22085
22679
|
const scope = new Set();
|
|
22086
22680
|
if (guests?.enabled !== true)
|
|
22087
22681
|
return scope;
|
|
22682
|
+
// #2348: with explicit grants, what an admin granted this account — of the instances that exist now (fleet.yaml's, or
|
|
22683
|
+
// a ClassicBot room's) — and nothing else: no connection access, no room membership, no open_instances.
|
|
22684
|
+
if (guests.grants !== undefined) {
|
|
22685
|
+
const key = `${owner.adapterId}:${owner.userId}`;
|
|
22686
|
+
const grant = Object.hasOwn(guests.grants, key) ? guests.grants[key] : undefined;
|
|
22687
|
+
if (!grant || !Array.isArray(grant.instances))
|
|
22688
|
+
return scope;
|
|
22689
|
+
const rooms = new Set((this.classicChannels?.getAll() ?? []).map((r) => r.instanceName));
|
|
22690
|
+
for (const name of grant.instances) {
|
|
22691
|
+
if ((this.fleetConfig?.instances && Object.hasOwn(this.fleetConfig.instances, name)) || rooms.has(name))
|
|
22692
|
+
scope.add(name);
|
|
22693
|
+
}
|
|
22694
|
+
return scope;
|
|
22695
|
+
}
|
|
22088
22696
|
const own = (map, key) => (map && Object.hasOwn(map, key) ? map[key] : undefined);
|
|
22089
22697
|
const world = this.worlds.get(owner.adapterId);
|
|
22090
22698
|
if (world && world.accessManager.isAllowed(owner.userId)) {
|
|
@@ -22126,7 +22734,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
22126
22734
|
const owner = record.owner;
|
|
22127
22735
|
if (this.guestScope(owner).size)
|
|
22128
22736
|
return "ok";
|
|
22129
|
-
|
|
22737
|
+
// #2348: with explicit grants no room's membership can give anything: nothing granted is gone, not pending. Nor does
|
|
22738
|
+
// a connection's start matter: grants are config (and ClassicBot rooms), read without the adapter.
|
|
22739
|
+
if (owner && this.fleetConfig?.web?.guests?.enabled === true && this.fleetConfig.web.guests.grants === undefined) {
|
|
22130
22740
|
// Before adapters initialize, topic scope is unknown, not revoked. Keep the returning cookie but grant no
|
|
22131
22741
|
// request until its configured world exists. Removed connections and initialized denied access stay final.
|
|
22132
22742
|
const channels = this.fleetConfig.channels ?? (this.fleetConfig.channel ? [this.fleetConfig.channel] : []);
|