@songsid/agend 2.1.7-beta.3 → 2.1.7-beta.5

package/dist/fleet-manager.js

@@ -1,5 +1,5 @@
 import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
-import { randomBytes } from "node:crypto";
+import { randomBytes, randomUUID } from "node:crypto";
 import { spawnSync } from "node:child_process";
 import { freemem, totalmem, cpus } from "node:os";
 import { createServer } from "node:http";
@@ -54,6 +54,12 @@ import { InstanceLifecycle, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } f
 import { TopicArchiver } from "./topic-archiver.js";
 import { StatuslineWatcher } from "./statusline-watcher.js";
 import { outboundHandlers } from "./outbound-handlers.js";
+import { DeliveryOutbox } from "./delivery-outbox.js";
+// The target Daemon can legitimately wait 30 minutes for a busy pane before
+// deciding to defer before submission. This is an alert threshold only: the
+// active manager/target generation pair owns the lane until the daemon reports
+// a state transition or that generation is replaced.
+export const DURABLE_DELIVERY_LANE_ALERT_MS = 35 * 60_000;
 import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
 import { handleViewRequest, isViewPath } from "./view-api.js";
 import { filterUsageProviders, formatDiscordUsageActivity, getUsageSnapshot, handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
@@ -290,6 +296,14 @@ export class FleetManager {
     spawnGate;
     /** Fleet-level backend reachability memory (fed by pty_error / startup panes). */
     backendOutage = new BackendOutageTracker();
+    /** Stable for this FleetManager OS-process lifetime; Daemon objects have their own boot IDs. */
+    managerBootId = randomUUID();
+    deliveryOutbox = null;
+    deliveryOutboxRecovered = false;
+    deliveryPumpScheduled = false;
+    deliveryPumpRunning = false;
+    deliveryPumpTimer = null;
+    activeDurableTargets = new Set();
     /** Live view of lifecycle.daemons — used throughout; not deprecated. */
     get daemons() { return this.lifecycle.daemons; }
     fleetConfig = null;
@@ -558,6 +572,243 @@ export class FleetManager {
         this.topicArchiver = new TopicArchiver(this);
         this.statuslineWatcher = new StatuslineWatcher(this);
     }
+    ensureDeliveryOutbox() {
+        if (this.deliveryOutbox)
+            return;
+        const outbox = new DeliveryOutbox(join(this.dataDir, "delivery-outbox.db"), this.managerBootId);
+        this.deliveryOutbox = outbox;
+        if (!this.deliveryOutboxRecovered) {
+            const recovered = outbox.recoverForBoot(this.managerBootId);
+            this.logger.info({ ...recovered }, "Recovered durable delivery outbox for this process boot");
+            this.deliveryOutboxRecovered = true;
+        }
+        outbox.on("admitted", () => this.scheduleDeliveryOutboxPump());
+        outbox.on("state", (event) => {
+            this.scheduleDeliveryOutboxPump();
+            if (event.deliveryId && (event.state === "failed" || event.state === "uncertain")) {
+                const row = outbox.get(event.deliveryId);
+                if (row && row.kind !== "delivery_outcome_notice" && row.kind !== "post_restart_outcome_notice") {
+                    const notice = `Durable delivery ${row.state}: operation_id=${row.operationId}; target=${row.targetInstance}; delivery_id=${row.deliveryId}.`;
+                    this.logger.error({ deliveryId: row.deliveryId, operationId: row.operationId, target: row.targetInstance, state: row.state }, notice);
+                    this.notifyFleetError(notice);
+                }
+            }
+        });
+        outbox.on("generation_recovered", () => this.scheduleDeliveryOutboxPump());
+        outbox.on("expired", (event) => {
+            this.scheduleDeliveryOutboxPump();
+            if (event.count)
+                this.notifyFleetError(`${event.count} durable delivery row(s) reached the 24-hour outbox bound; ${event.uncertain ?? 0} unresolved submissions are uncertain and sender notices were queued.`);
+        });
+        this.scheduleDeliveryOutboxPump();
+    }
+    /** Synchronous SQLite admission; the caller may acknowledge only after this returns. */
+    admitDurableDelivery(input) {
+        const outbox = this.deliveryOutbox;
+        if (!outbox)
+            throw new Error("Durable delivery store is unavailable");
+        const source = this.daemons.get(input.sourceInstance);
+        const sourceDaemonBootId = input.sourceDaemonBootId ?? source?.bootId;
+        if (!source || !sourceDaemonBootId || source.bootId !== sourceDaemonBootId) {
+            throw new Error("Source daemon generation is no longer current; outcome is unknown");
+        }
+        const sourceKey = `mcp:${input.sourceInstance}:${input.operationId}:${input.targetInstance}:${input.targetSession ?? ""}:${input.kind}`;
+        const admitted = outbox.admit({ ...input, sourceDaemonBootId, sourceKey });
+        if (admitted.inserted)
+            this.logger.info({
+                deliveryId: admitted.delivery.deliveryId,
+                operationId: input.operationId,
+                source: input.sourceInstance,
+                target: input.targetInstance,
+                kind: input.kind,
+            }, "Durable cross-instance delivery accepted");
+        this.scheduleDeliveryOutboxPump();
+        return {
+            deliveryId: admitted.delivery.deliveryId,
+            state: admitted.delivery.state,
+            duplicate: !admitted.inserted,
+        };
+    }
+    queryDurableDeliveryStatus(callerInstance, selector) {
+        this.ensureDeliveryOutbox();
+        return this.deliveryOutbox.queryStatusForInstance(callerInstance, selector);
+    }
+    getDaemonBootId(instanceName) {
+        return this.daemons.get(instanceName)?.bootId;
+    }
+    markDurableResponseDelivered(sourceInstance, operationId) {
+        this.deliveryOutbox?.markResponseDelivered(sourceInstance, operationId);
+    }
+    /** Called after a replacement Daemon object is installed in lifecycle.daemons. */
+    onDaemonReady(name, daemonBootId) {
+        const outbox = this.deliveryOutbox;
+        if (!outbox)
+            return;
+        for (const prior of outbox.getUnansweredAccepted(name, daemonBootId)) {
+            try {
+                outbox.admitPostRestartOutcomeNotice(prior, daemonBootId);
+            }
+            catch (err) {
+                this.logger.error({ err, source: name, operationId: prior.operationId }, "Could not persist post-restart delivery outcome notice");
+            }
+        }
+        this.scheduleDeliveryOutboxPump();
+    }
+    scheduleDeliveryOutboxPump(delayMs = 0) {
+        if (!this.deliveryOutbox || this.shuttingDown)
+            return;
+        if (this.deliveryPumpTimer) {
+            if (delayMs > 0)
+                return;
+            clearTimeout(this.deliveryPumpTimer);
+            this.deliveryPumpTimer = null;
+        }
+        if (this.deliveryPumpRunning) {
+            this.deliveryPumpScheduled = true;
+            return;
+        }
+        this.deliveryPumpTimer = setTimeout(() => {
+            this.deliveryPumpTimer = null;
+            void this.runDeliveryOutboxPump();
+        }, Math.max(0, delayMs));
+        this.deliveryPumpTimer.unref?.();
+    }
+    async runDeliveryOutboxPump() {
+        const outbox = this.deliveryOutbox;
+        if (!outbox || this.shuttingDown || this.deliveryPumpRunning)
+            return;
+        this.deliveryPumpRunning = true;
+        this.deliveryPumpScheduled = false;
+        try {
+            outbox.expireStale();
+            while (this.activeDurableTargets.size < 8) {
+                const claimed = outbox.claimNext(this.managerBootId, target => this.isInstanceRestarting(target) ? null : this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets);
+                if (!claimed)
+                    break;
+                this.activeDurableTargets.add(claimed.targetInstance);
+                void this.dispatchDurableDelivery(claimed).finally(() => {
+                    this.activeDurableTargets.delete(claimed.targetInstance);
+                    this.scheduleDeliveryOutboxPump();
+                });
+            }
+        }
+        catch (err) {
+            this.logger.error({ err }, "Durable delivery dispatcher failed; committed rows remain in SQLite");
+            this.scheduleDeliveryOutboxPump(5_000);
+        }
+        finally {
+            this.deliveryPumpRunning = false;
+            if (this.deliveryPumpScheduled)
+                this.scheduleDeliveryOutboxPump();
+            const nextRetryAt = outbox.nextRetryAt();
+            if (nextRetryAt && !this.deliveryPumpTimer) {
+                this.scheduleDeliveryOutboxPump(Math.max(1, Date.parse(nextRetryAt) - Date.now()));
+            }
+            const nextExpiryAt = outbox.nextExpiryAt();
+            if (nextExpiryAt && !this.deliveryPumpTimer) {
+                this.scheduleDeliveryOutboxPump(Math.max(1, Date.parse(nextExpiryAt) - Date.now()));
+            }
+        }
+    }
+    async dispatchDurableDelivery(claimed) {
+        const outbox = this.deliveryOutbox;
+        if (!outbox)
+            return;
+        const target = claimed.targetInstance;
+        const attempt = claimed.attemptNo;
+        const payload = { ...claimed.payload };
+        const laneReleased = this.waitForDurableLaneRelease(claimed);
+        const rawMeta = payload.meta && typeof payload.meta === "object"
+            ? payload.meta
+            : {};
+        // Arm the visible liveness alert as soon as this generation owns the lane,
+        // including time spent waiting for a handoff/idle gate to return.
+        try {
+            let sent;
+            if (claimed.kind === "raw_paste") {
+                if (payload.type !== "raw_paste" || typeof payload.content !== "string") {
+                    throw new Error("Durable raw_paste row has an invalid payload");
+                }
+                // Preserve the schedule's exact bytes and raw route. The target daemon
+                // owns readiness, paste, one Enter, and the fenced outbox transition.
+                sent = await this.deliverToInstance(target, {
+                    type: "raw_paste",
+                    content: payload.content,
+                    delivery_id: claimed.deliveryId,
+                    delivery_attempt: String(claimed.attemptNo),
+                }, { waitForIdle: false });
+            }
+            else {
+                payload.meta = {
+                    ...rawMeta,
+                    delivery_id: claimed.deliveryId,
+                    delivery_attempt: String(claimed.attemptNo),
+                    from_instance: typeof rawMeta.from_instance === "string" && rawMeta.from_instance
+                        ? rawMeta.from_instance
+                        : claimed.sourceInstance,
+                    correlation_id: claimed.correlationId ?? String(rawMeta.correlation_id ?? ""),
+                };
+                sent = await this.deliverToInstance(target, payload, {
+                    isCrossInstance: true,
+                    waitForIdle: claimed.kind !== "steer",
+                });
+            }
+            if (!sent) {
+                const current = outbox.get(claimed.deliveryId);
+                if (current?.state === "delivering") {
+                    outbox.retryBeforeBegin(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, "delivery was cancelled before target IPC handoff", Math.min(60_000, 1_000 * 2 ** Math.min(attempt - 1, 6)));
+                }
+            }
+        }
+        catch (err) {
+            const latest = outbox.get(claimed.deliveryId);
+            const reason = err instanceof Error ? err.message : String(err);
+            if (latest?.state === "submission_started") {
+                outbox.complete(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, "uncertain", reason);
+            }
+            else {
+                outbox.retryBeforeBegin(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, reason, Math.min(60_000, 1_000 * 2 ** Math.min(attempt - 1, 6)));
+            }
+            this.logger.warn({ deliveryId: claimed.deliveryId, target, attempt, err: reason }, "Durable delivery dispatch deferred");
+        }
+        await laneReleased;
+    }
+    async waitForDurableLaneRelease(claimed) {
+        const outbox = this.deliveryOutbox;
+        if (!outbox)
+            return;
+        const deliveryId = claimed.deliveryId;
+        const released = (row) => !row
+            || row.state === "queued" || row.state === "retry_wait"
+            || row.state === "delivered" || row.state === "failed"
+            || row.state === "uncertain" || row.state === "cancelled";
+        if (released(outbox.get(deliveryId)))
+            return;
+        await new Promise(resolve => {
+            const finish = () => {
+                clearTimeout(alertTimer);
+                outbox.off("state", check);
+                outbox.off("generation_recovered", check);
+                resolve();
+            };
+            const check = () => {
+                if (released(outbox.get(deliveryId)))
+                    finish();
+            };
+            const alertTimer = setTimeout(() => {
+                const current = outbox.get(deliveryId);
+                if ((current?.state === "delivering" || current?.state === "submission_started")
+                    && current.targetDaemonBootId === claimed.targetDaemonBootId
+                    && current.managerBootId === claimed.managerBootId) {
+                    this.notifyFleetError(`Durable delivery to ${claimed.targetInstance} is still waiting after ${Math.round(DURABLE_DELIVERY_LANE_ALERT_MS / 60_000)} minutes; the same daemon generation retains delivery ${deliveryId}'s lane until it reports or is replaced.`);
+                }
+            }, DURABLE_DELIVERY_LANE_ALERT_MS);
+            alertTimer.unref?.();
+            outbox.on("state", check);
+            outbox.on("generation_recovered", check);
+            check();
+        });
+    }
     spawnConcurrency() {
         const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
         if (explicit != null)
@@ -1509,16 +1760,16 @@ export class FleetManager {
     }
     async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch) {
         if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-            return;
+            return false;
         await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
         if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-            return;
+            return false;
         let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
         if (this.lifecycle.isPaused(instanceName)) {
             const wakeStartedAt = Date.now();
             await this.lifecycle.wake(instanceName, 30_000);
             if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-                return;
+                return false;
             // Waking added one to the warm count — make room by evicting a different
             // LRU idle instance (never this one; it's about to work).
             this.enforceWarmCap(instanceName);
@@ -1528,20 +1779,23 @@ export class FleetManager {
         const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter, () => !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch));
         if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
             this.logger.info({ instanceName }, "Pending delivery dropped by user cancel");
-            return;
+            return false;
         }
         // A server crash can land while waitForInstanceIdle is pending. Re-check
         // immediately before the old timeout path would force text into a boot UI.
         await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
         if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-            return;
+            return false;
         if (!idle) {
             this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
         }
-        await this.sendWhenConnected(instanceName, payload, deliveryEpoch);
-        if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-            return;
-        this.lastDeliveryAt.set(instanceName, Date.now());
+        const sent = await this.sendWhenConnected(instanceName, payload, deliveryEpoch);
+        if (!sent)
+            return false;
+        if (this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
+            this.lastDeliveryAt.set(instanceName, Date.now());
+        }
+        return true;
     }
     async holdDeliveryForStorm(instanceName, deliveryEpoch) {
         if (!this.stormWindow.isDeliveryHeld(instanceName))
@@ -1567,26 +1821,26 @@ export class FleetManager {
      */
     async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName)) {
         if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-            return;
+            return false;
         const queued = this.ipcWaitTails.get(instanceName);
         if (!queued) {
             const ipc = this.instanceIpcClients.get(instanceName);
             if (ipc?.connected && ipc.send(payload))
-                return;
+                return true;
         }
         const attempt = (queued ?? Promise.resolve())
             .catch(() => { })
             .then(() => {
             if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-                return;
+                return false;
             return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch);
         });
         // The chain stores a settled-either-way promise so one failed delivery cannot
         // wedge every later one, and so `queued` above is safe to await unguarded.
-        const tail = attempt.catch(() => { });
+        const tail = attempt.then(() => { }, () => { });
         this.ipcWaitTails.set(instanceName, tail);
         try {
-            await attempt;
+            return await attempt;
         }
         finally {
             // Only the last waiter clears the chain; while a queue is still draining the
@@ -1602,12 +1856,12 @@ export class FleetManager {
         let warned = false;
         for (;;) {
             if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-                return;
+                return false;
             // Re-read every round: a reconnect replaces the IpcClient object entirely,
             // so a cached reference would stay dead forever.
             const ipc = this.instanceIpcClients.get(instanceName);
             if (ipc?.connected && ipc.send(payload))
-                return;
+                return true;
             if (Date.now() >= deadline) {
                 throw new Error(`Instance '${instanceName}' IPC is unavailable`);
             }
@@ -1638,22 +1892,24 @@ export class FleetManager {
             if (this.lifecycle.isPaused(instanceName)) {
                 await this.lifecycle.wake(instanceName, 30_000);
                 if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-                    return;
+                    return false;
                 this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
             }
-            await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch);
-            if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
-                return;
+            const sent = await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch);
+            if (!sent)
+                return false;
             // A cross-instance item arriving before the daemon observes this turn as
             // working must not trust the stale idle snapshot from before the send.
-            this.lastDeliveryAt.set(instanceName, Date.now());
-            return;
+            if (this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
+                this.lastDeliveryAt.set(instanceName, Date.now());
+            }
+            return true;
         }
         const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
         const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch));
         this.idleGatedDeliveryTails.set(instanceName, delivery);
         try {
-            await delivery;
+            return await delivery;
         }
         finally {
             if (this.idleGatedDeliveryTails.get(instanceName) === delivery) {
@@ -2279,10 +2535,16 @@ export class FleetManager {
             workingDirectory,
             reason: "restart",
         }, () => this.doRestartSingleInstance(name, opts))
-            .finally(() => this.restartsInFlight.delete(name));
+            .finally(() => {
+            this.restartsInFlight.delete(name);
+            this.scheduleDeliveryOutboxPump();
+        });
         this.restartsInFlight.set(name, run);
         return run;
     }
+    isInstanceRestarting(name) {
+        return this.restartsInFlight.has(name);
+    }
     async doRestartSingleInstance(name, opts) {
         if (this.configPath) {
             this.loadConfig(this.configPath);
@@ -2575,6 +2837,7 @@ export class FleetManager {
         this.shuttingDown = false;
         this.configPath = configPath;
         this.loadEnvFile();
+        this.ensureDeliveryOutbox();
         // Rotate fleet.log if oversized (before any logging)
         rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
         const fleet = this.loadConfig(configPath);
@@ -2797,7 +3060,7 @@ export class FleetManager {
                 ...DEFAULT_SCHEDULER_CONFIG,
                 ...this.fleetConfig?.defaults.scheduler,
             };
-            this.scheduler = new Scheduler(join(this.dataDir, "scheduler.db"), (schedule) => this.handleScheduleTrigger(schedule), schedulerConfig, (name) => this.fleetConfig?.instances?.[name] != null || !!this.classicChannels?.getAll().some(ch => ch.instanceName === name));
+            this.scheduler = new Scheduler(join(this.dataDir, "scheduler.db"), (schedule, runId) => this.handleScheduleTrigger(schedule, runId), schedulerConfig, (name) => this.fleetConfig?.instances?.[name] != null || !!this.classicChannels?.getAll().some(ch => ch.instanceName === name));
             this.scheduler.init();
             this.logger.info("Scheduler initialized");
             // Tips share the daily-report clock but remain an independent internal
@@ -5197,6 +5460,8 @@ export class FleetManager {
         const requestId = msg.requestId;
         const fleetRequestId = msg.fleetRequestId;
         const senderSessionName = msg.senderSessionName;
+        const operationId = typeof msg.operationId === "string" ? msg.operationId : undefined;
+        const sourceDaemonBootId = typeof msg.sourceDaemonBootId === "string" ? msg.sourceDaemonBootId : undefined;
         const respond = (result, error) => {
             const ipc = this.instanceIpcClients.get(instanceName);
             let sent = false;
@@ -5314,7 +5579,9 @@ export class FleetManager {
         // Dispatch fleet-specific tools via handler map
         const handler = outboundHandlers.get(tool);
         if (handler) {
-            await handler(this, args, respond, { instanceName, requestId, fleetRequestId, senderSessionName });
+            await handler(this, args, respond, {
+                instanceName, requestId, fleetRequestId, senderSessionName, operationId, sourceDaemonBootId,
+            });
         }
         else {
             respond(null, `Unknown tool: ${tool}`);
@@ -5350,7 +5617,7 @@ export class FleetManager {
         }
     }
     // ===================== Scheduler =====================
-    async handleScheduleTrigger(schedule) {
+    async handleScheduleTrigger(schedule, stableRunId = randomUUID()) {
         const { target, reply_chat_id, reply_thread_id, message, label, id, source, silent } = schedule;
         const RATE_LIMIT_DEFER_THRESHOLD = 85;
         const rl = this.statuslineWatcher.getRateLimits(target);
@@ -5368,20 +5635,32 @@ export class FleetManager {
         }
         // Silent mode: paste directly to tmux pane — no channel message.
         if (silent) {
-            const ipc = this.instanceIpcClients.get(target);
-            if (ipc) {
-                ipc.send({
+            // The outbox commit is the schedule's durable acceptance point. The run
+            // key is the scheduled fire instant (or an invocation UUID for a manual
+            // trigger), so restart/catch-up reuses the same raw paste row.
+            this.ensureDeliveryOutbox();
+            const runKey = encodeURIComponent(stableRunId);
+            const sourceKey = `schedule:${encodeURIComponent(id)}:${runKey}:${encodeURIComponent(target)}:raw_paste`;
+            const operationId = `schedule:${id}:${stableRunId}`;
+            const sourceBootId = this.daemons.get(source)?.bootId ?? this.managerBootId;
+            const admitted = this.deliveryOutbox.admit({
+                operationId,
+                sourceKey,
+                sourceInstance: source,
+                sourceDaemonBootId: sourceBootId,
+                targetInstance: target,
+                kind: "raw_paste",
+                correlationId: operationId,
+                payload: {
                     type: "raw_paste",
                     content: message,
-                    delivery_epoch: this.getDeliveryEpoch(target),
-                });
-                this.scheduler.recordRun(id, "delivered");
-                this.logger.info({ target, scheduleId: id, label }, "Silent schedule injected via raw_paste");
-            }
-            else {
-                this.scheduler.recordRun(id, "instance_offline", "IPC not connected");
-                this.logger.warn({ target, scheduleId: id }, "Silent schedule: IPC not connected, skipping");
-            }
+                    schedule_id: id,
+                    schedule_run_id: stableRunId,
+                },
+            });
+            this.scheduler.recordRun(id, "queued", `durable raw_paste delivery_id=${admitted.delivery.deliveryId}`);
+            this.logger.info({ target, scheduleId: id, runId: stableRunId, deliveryId: admitted.delivery.deliveryId, duplicate: !admitted.inserted }, "Silent schedule durably admitted as raw_paste");
+            this.scheduleDeliveryOutboxPump();
             return;
         }
         const schedulerDefaults = this.fleetConfig?.defaults.scheduler;
@@ -6528,6 +6807,8 @@ export class FleetManager {
      * the chain drains, so idle messages hold no memory.
      */
     deliveryStatusChains = new Map();
+    /** #725: in-flight CLI env probes, keyed by backend name. Coalesces concurrent /model requests. */
+    pendingCliEnvProbes = new Map();
     reactMessageStatus(instanceName, chatId, messageId, emoji, threadId) {
         // React via the adapter BOUND to this instance — NOT the first discord world.
         // Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
@@ -6584,9 +6865,10 @@ export class FleetManager {
                 return;
             // Thread-aware adapters (Discord) react where the thread is; Telegram
             // addresses the supergroup chat and ignores the thread part.
-            // Discord needs the old reaction removed before the new one is added.
-            // Telegram's setMessageReaction replaces the bot's one current reaction.
-            if (prev && prev !== emoji && adapter.unreact && !(adapter instanceof TelegramAdapter)) {
+            // #972: add-only by default — only remove ❌ when leaving the failed
+            // state. 👀/⏳/✅ stacking together is harmless; leaving ❌ visible
+            // after a recovery would be misleading.
+            if (prev === "❌" && adapter.unreact && !(adapter instanceof TelegramAdapter)) {
                 // Best effort: a failed removal must not block the new status.
                 await adapter.unreact(target, messageId, prev, threadId).catch(e => this.logger.debug({ err: e.message }, "Delivery status reaction removal failed"));
             }
@@ -9901,6 +10183,15 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
      * previous list, never stall the command that asked for it.
      */
     async probeBackendBounded(backend, opts = {}) {
+        // #725: single-flight per backend — if a probe for this backend is already
+        // in flight, return the same promise instead of starting a duplicate.
+        // Refresh probes (refreshVendorCatalog=true) are always started fresh since
+        // the caller explicitly asked for a new vendor fetch.
+        if (!opts.refreshVendorCatalog) {
+            const existing = this.pendingCliEnvProbes.get(backend);
+            if (existing)
+                return existing;
+        }
         const work = this.probeBackend(backend, opts);
         work.catch(() => { });
         let timer;
@@ -9910,12 +10201,14 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 resolve(null);
             }, CLI_ENV_PROBE_DEADLINE_MS);
         });
-        try {
-            return await Promise.race([work, deadline]);
-        }
-        finally {
+        const bounded = Promise.race([work, deadline]).finally(() => {
             clearTimeout(timer);
-        }
+            if (this.pendingCliEnvProbes.get(backend) === bounded)
+                this.pendingCliEnvProbes.delete(backend);
+        });
+        if (!opts.refreshVendorCatalog)
+            this.pendingCliEnvProbes.set(backend, bounded);
+        return bounded;
     }
     /**
      * Resolve the effective model for a fleet or ClassicBot instance, plus where it
@@ -10025,8 +10318,10 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 backends.add(ch.backend);
         if (backends.size === 0)
             backends.add("claude-code");
+        // #724: use the bounded variant so background startup probes cannot hang
+        // indefinitely — each is wrapped by CLI_ENV_PROBE_DEADLINE_MS.
         for (const b of backends)
-            void this.probeBackend(b);
+            void this.probeBackendBounded(b);
     }
     /** Best-effort model list for `/model`: cached CLI env first, else live probe. Never throws. */
     async getModelOptions(instanceName, refresh = false, onLiveProbe) {
@@ -10448,7 +10743,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
         const cached = this.readCliEnv("claude-code");
         if (cached?.apiModels?.length)
             return cached.apiModels;
-        const env = await this.probeBackend("claude-code");
+        const env = await this.probeBackendBounded("claude-code");
         return env?.apiModels ?? [];
     }
     /** Replace a consumed "/model" menu with the full account catalog tier. */