@songsid/agend 2.1.0-beta.27 → 2.1.0-beta.28

package/dist/fleet-manager.js

@@ -41,6 +41,7 @@ import { setLocale, detectLocale, t } from "./locale.js";
 import { handleAgentRequest } from "./agent-endpoint.js";
 import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
 import { readLastInboundAt } from "./daemon.js";
+import { clearPausedMarker } from "./pause-marker.js";
 import { getTmuxSession } from "./config.js";
 export function resolveReplyThreadId(argsThreadId, instanceConfig) {
     if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
@@ -101,6 +102,12 @@ export class FleetManager {
     lastActivity = new Map();
     /** Latest pane-derived execution snapshot reported by each daemon. */
     instanceStateCache = new Map();
+    /** Per-instance tail keeps cross-instance and scheduled deliveries FIFO. */
+    idleGatedDeliveryTails = new Map();
+    /** Non-user work must observe a fresh idle snapshot after the latest delivery. */
+    lastDeliveryAt = new Map();
+    /** State-cache updates wake event-driven idle waiters without busy polling. */
+    instanceIdleWaiters = new Map();
     lastInboundUser = new Map(); // instanceName → last username
     // Active "🛑 Cancel" buttons, tracked per button (keyed by messageId) rather
     // than one-per-instance. A button is retired (deleted, with bounded retry) on
@@ -370,16 +377,100 @@ export class FleetManager {
             observedAt: numberOr(msg.observedAt, now),
             stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
         });
+        for (const check of this.instanceIdleWaiters.get(name) ?? [])
+            check();
+    }
+    waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0) {
+        const isReady = () => {
+            const snapshot = this.instanceStateCache.get(instanceName);
+            return snapshot?.state === "idle"
+                && (idleObservedAfter === 0 || snapshot.observedAt > idleObservedAfter);
+        };
+        if (isReady())
+            return Promise.resolve(true);
+        return new Promise(resolve => {
+            let settled = false;
+            const finish = (idle) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timeout);
+                clearInterval(queryTimer);
+                const waiters = this.instanceIdleWaiters.get(instanceName);
+                waiters?.delete(check);
+                if (waiters?.size === 0)
+                    this.instanceIdleWaiters.delete(instanceName);
+                resolve(idle);
+            };
+            const check = () => { if (isReady())
+                finish(true); };
+            const query = () => {
+                const ipc = this.instanceIpcClients.get(instanceName);
+                if (ipc?.connected) {
+                    ipc.send({ type: "query_instance_state", requestId: `idle-gate-${Date.now()}` });
+                }
+                check();
+            };
+            const waiters = this.instanceIdleWaiters.get(instanceName) ?? new Set();
+            waiters.add(check);
+            this.instanceIdleWaiters.set(instanceName, waiters);
+            const timeout = setTimeout(() => finish(false), timeoutMs);
+            const queryTimer = setInterval(query, 1_000);
+            query();
+        });
     }
-    /** Single delivery facade: wake an auto-paused CLI before sending to daemon IPC. */
-    async deliverToInstance(instanceName, payload) {
+    async deliverWithIdleGate(instanceName, payload, timeoutMs) {
+        let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
         if (this.lifecycle.isPaused(instanceName)) {
+            const wakeStartedAt = Date.now();
             await this.lifecycle.wake(instanceName, 30_000);
+            // Never satisfy a post-wake gate from a stale pre-pause cache entry.
+            idleObservedAfter = Math.max(idleObservedAfter, wakeStartedAt);
+        }
+        const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter);
+        if (!idle) {
+            this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
         }
         const ipc = this.instanceIpcClients.get(instanceName);
         if (!ipc?.connected)
             throw new Error(`Instance '${instanceName}' IPC is unavailable`);
         ipc.send(payload);
+        this.lastDeliveryAt.set(instanceName, Date.now());
+    }
+    /** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
+    async deliverToInstance(instanceName, payload, options = {}) {
+        const meta = payload.meta && typeof payload.meta === "object"
+            ? payload.meta
+            : undefined;
+        const inferredCrossInstance = (typeof meta?.from_instance === "string" && meta.from_instance.length > 0)
+            || meta?.is_cross_instance === true
+            || payload.is_cross_instance === true;
+        const waitForIdle = options.waitForIdle
+            ?? ((options.isCrossInstance ?? inferredCrossInstance) || payload.type === "fleet_schedule_trigger");
+        if (!waitForIdle) {
+            if (this.lifecycle.isPaused(instanceName)) {
+                await this.lifecycle.wake(instanceName, 30_000);
+            }
+            const ipc = this.instanceIpcClients.get(instanceName);
+            if (!ipc?.connected)
+                throw new Error(`Instance '${instanceName}' IPC is unavailable`);
+            ipc.send(payload);
+            // A cross-instance item arriving before the daemon observes this turn as
+            // working must not trust the stale idle snapshot from before the send.
+            this.lastDeliveryAt.set(instanceName, Date.now());
+            return;
+        }
+        const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
+        const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, payload, options.idleTimeoutMs ?? 60_000));
+        this.idleGatedDeliveryTails.set(instanceName, delivery);
+        try {
+            await delivery;
+        }
+        finally {
+            if (this.idleGatedDeliveryTails.get(instanceName) === delivery) {
+                this.idleGatedDeliveryTails.delete(instanceName);
+            }
+        }
     }
     /** Fleet admin is an explicit config allowlist entry, not merely an open/paired user. */
     isFleetAdmin(userId, adapterId) {
@@ -411,6 +502,10 @@ export class FleetManager {
         await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model));
     }
     async startInstance(name, config, topicMode) {
+        if (this.lifecycle.isPaused(name)) {
+            this.logger.info({ name }, "Persisted paused instance — skipping startup");
+            return;
+        }
         if (config.general_topic) {
             this.ensureGeneralInstructions(config.working_directory, config.backend);
         }
@@ -418,6 +513,20 @@ export class FleetManager {
         // Auto-connect IPC — daemon.start() ensures socket is ready before resolving
         await this.connectIpcToInstance(name);
     }
+    /** Recreate a daemon for a marker-only paused instance after an explicit wake/delivery. */
+    async startPersistedPausedInstance(name) {
+        const topicMode = this.fleetConfig?.channel?.mode === "topic"
+            || !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
+        const fleetConfig = this.fleetConfig?.instances[name];
+        if (fleetConfig) {
+            await this.startInstance(name, fleetConfig, topicMode);
+            return;
+        }
+        const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
+        if (!channel || !this.classicChannels)
+            throw new Error(`Paused instance '${name}' is no longer configured`);
+        await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model));
+    }
     /**
      * Start instances with configurable concurrency and stagger delay.
      * Instances sharing the same working_directory are serialized within a group
@@ -478,6 +587,7 @@ export class FleetManager {
     async stopInstance(name) {
         this.failoverActive.delete(name);
         this.instanceStateCache.delete(name);
+        this.lastDeliveryAt.delete(name);
         return this.lifecycle.stop(name);
     }
     /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
@@ -2375,7 +2485,7 @@ export class FleetManager {
                     type: "fleet_schedule_trigger",
                     payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
                     meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
-                });
+                }, { waitForIdle: true });
                 // A scheduled trigger also puts the instance to work — show a cancel button.
                 void this.sendCancelButton(target);
                 return true;
@@ -2943,6 +3053,8 @@ export class FleetManager {
         }
     }
     startStatuslineWatcher(name) {
+        if (this.lifecycle.isPaused(name))
+            return;
         this.statuslineWatcher.watch(name);
     }
     stopStatuslineWatcher(name) {
@@ -4045,6 +4157,7 @@ When users create specialized instances, suggest these configurations:
         if (!classicChannels)
             return "Classic channel manager not initialized.";
         const instanceName = classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
+        clearPausedMarker(this.getInstanceDir(instanceName));
         const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
         classicChannels.register(channelId, adapterId, instanceName, channelName || channelId, userId, selectedBackend);
         // Bind this classic instance to the bot that started it (authoritative), so
@@ -4070,6 +4183,7 @@ When users create specialized instances, suggest these configurations:
             return t("classic.no_agent");
         this.instanceWorldBinding.delete(ch.instanceName);
         await this.stopInstance(ch.instanceName).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to stop classic instance"));
+        clearPausedMarker(this.getInstanceDir(ch.instanceName));
         this.reregisterClassicChannels();
         this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
         return t("classic.stopped");