@songsid/agend 2.1.0-beta.27 → 2.1.0-beta.29

package/dist/fleet-manager.js

@@ -41,6 +41,7 @@ import { setLocale, detectLocale, t } from "./locale.js";
 import { handleAgentRequest } from "./agent-endpoint.js";
 import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
 import { readLastInboundAt } from "./daemon.js";
+import { clearPausedMarker } from "./pause-marker.js";
 import { getTmuxSession } from "./config.js";
 export function resolveReplyThreadId(argsThreadId, instanceConfig) {
     if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
@@ -101,6 +102,12 @@ export class FleetManager {
     lastActivity = new Map();
     /** Latest pane-derived execution snapshot reported by each daemon. */
     instanceStateCache = new Map();
+    /** Per-instance tail keeps cross-instance and scheduled deliveries FIFO. */
+    idleGatedDeliveryTails = new Map();
+    /** Non-user work must observe a fresh idle snapshot after the latest delivery. */
+    lastDeliveryAt = new Map();
+    /** State-cache updates wake event-driven idle waiters without busy polling. */
+    instanceIdleWaiters = new Map();
     lastInboundUser = new Map(); // instanceName → last username
     // Active "🛑 Cancel" buttons, tracked per button (keyed by messageId) rather
     // than one-per-instance. A button is retired (deleted, with bounded retry) on
@@ -357,6 +364,9 @@ export class FleetManager {
             return null;
         return this.instanceStateCache.get(name)?.state ?? null;
     }
+    isClassicInstance(name) {
+        return this.classicChannels?.getAll().some(channel => channel.instanceName === name) ?? false;
+    }
     cacheInstanceExecutionState(name, msg) {
         const state = msg.state;
         if (state !== "idle" && state !== "working" && state !== "stuck")
@@ -370,16 +380,100 @@ export class FleetManager {
             observedAt: numberOr(msg.observedAt, now),
             stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
         });
+        for (const check of this.instanceIdleWaiters.get(name) ?? [])
+            check();
+    }
+    waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0) {
+        const isReady = () => {
+            const snapshot = this.instanceStateCache.get(instanceName);
+            return snapshot?.state === "idle"
+                && (idleObservedAfter === 0 || snapshot.observedAt > idleObservedAfter);
+        };
+        if (isReady())
+            return Promise.resolve(true);
+        return new Promise(resolve => {
+            let settled = false;
+            const finish = (idle) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timeout);
+                clearInterval(queryTimer);
+                const waiters = this.instanceIdleWaiters.get(instanceName);
+                waiters?.delete(check);
+                if (waiters?.size === 0)
+                    this.instanceIdleWaiters.delete(instanceName);
+                resolve(idle);
+            };
+            const check = () => { if (isReady())
+                finish(true); };
+            const query = () => {
+                const ipc = this.instanceIpcClients.get(instanceName);
+                if (ipc?.connected) {
+                    ipc.send({ type: "query_instance_state", requestId: `idle-gate-${Date.now()}` });
+                }
+                check();
+            };
+            const waiters = this.instanceIdleWaiters.get(instanceName) ?? new Set();
+            waiters.add(check);
+            this.instanceIdleWaiters.set(instanceName, waiters);
+            const timeout = setTimeout(() => finish(false), timeoutMs);
+            const queryTimer = setInterval(query, 1_000);
+            query();
+        });
     }
-    /** Single delivery facade: wake an auto-paused CLI before sending to daemon IPC. */
-    async deliverToInstance(instanceName, payload) {
+    async deliverWithIdleGate(instanceName, payload, timeoutMs) {
+        let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
         if (this.lifecycle.isPaused(instanceName)) {
+            const wakeStartedAt = Date.now();
             await this.lifecycle.wake(instanceName, 30_000);
+            // Never satisfy a post-wake gate from a stale pre-pause cache entry.
+            idleObservedAfter = Math.max(idleObservedAfter, wakeStartedAt);
+        }
+        const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter);
+        if (!idle) {
+            this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
         }
         const ipc = this.instanceIpcClients.get(instanceName);
         if (!ipc?.connected)
             throw new Error(`Instance '${instanceName}' IPC is unavailable`);
         ipc.send(payload);
+        this.lastDeliveryAt.set(instanceName, Date.now());
+    }
+    /** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
+    async deliverToInstance(instanceName, payload, options = {}) {
+        const meta = payload.meta && typeof payload.meta === "object"
+            ? payload.meta
+            : undefined;
+        const inferredCrossInstance = (typeof meta?.from_instance === "string" && meta.from_instance.length > 0)
+            || meta?.is_cross_instance === true
+            || payload.is_cross_instance === true;
+        const waitForIdle = options.waitForIdle
+            ?? ((options.isCrossInstance ?? inferredCrossInstance) || payload.type === "fleet_schedule_trigger");
+        if (!waitForIdle) {
+            if (this.lifecycle.isPaused(instanceName)) {
+                await this.lifecycle.wake(instanceName, 30_000);
+            }
+            const ipc = this.instanceIpcClients.get(instanceName);
+            if (!ipc?.connected)
+                throw new Error(`Instance '${instanceName}' IPC is unavailable`);
+            ipc.send(payload);
+            // A cross-instance item arriving before the daemon observes this turn as
+            // working must not trust the stale idle snapshot from before the send.
+            this.lastDeliveryAt.set(instanceName, Date.now());
+            return;
+        }
+        const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
+        const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, payload, options.idleTimeoutMs ?? 60_000));
+        this.idleGatedDeliveryTails.set(instanceName, delivery);
+        try {
+            await delivery;
+        }
+        finally {
+            if (this.idleGatedDeliveryTails.get(instanceName) === delivery) {
+                this.idleGatedDeliveryTails.delete(instanceName);
+            }
+        }
     }
     /** Fleet admin is an explicit config allowlist entry, not merely an open/paired user. */
     isFleetAdmin(userId, adapterId) {
@@ -408,9 +502,13 @@ export class FleetManager {
             return;
         await this.stopInstance(instanceName);
         await new Promise(resolve => setTimeout(resolve, 250));
-        await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model));
+        await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
     }
     async startInstance(name, config, topicMode) {
+        if (this.lifecycle.isPaused(name)) {
+            this.logger.info({ name }, "Persisted paused instance — skipping startup");
+            return;
+        }
         if (config.general_topic) {
             this.ensureGeneralInstructions(config.working_directory, config.backend);
         }
@@ -418,6 +516,20 @@ export class FleetManager {
         // Auto-connect IPC — daemon.start() ensures socket is ready before resolving
         await this.connectIpcToInstance(name);
     }
+    /** Recreate a daemon for a marker-only paused instance after an explicit wake/delivery. */
+    async startPersistedPausedInstance(name) {
+        const topicMode = this.fleetConfig?.channel?.mode === "topic"
+            || !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
+        const fleetConfig = this.fleetConfig?.instances[name];
+        if (fleetConfig) {
+            await this.startInstance(name, fleetConfig, topicMode);
+            return;
+        }
+        const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
+        if (!channel || !this.classicChannels)
+            throw new Error(`Paused instance '${name}' is no longer configured`);
+        await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+    }
     /**
      * Start instances with configurable concurrency and stagger delay.
      * Instances sharing the same working_directory are serialized within a group
@@ -478,6 +590,7 @@ export class FleetManager {
     async stopInstance(name) {
         this.failoverActive.delete(name);
         this.instanceStateCache.delete(name);
+        this.lastDeliveryAt.delete(name);
         return this.lifecycle.stop(name);
     }
     /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
@@ -588,9 +701,11 @@ export class FleetManager {
                 const fleetModel = this.fleetConfig?.defaults?.model;
                 const oldBackends = new Map();
                 const oldModels = new Map();
+                const oldAutoPause = new Map();
                 for (const ch of this.classicChannels.getAll()) {
                     oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
                     oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
+                    oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
                 }
                 if (!this.classicChannels.checkReload())
                     return;
@@ -598,14 +713,16 @@ export class FleetManager {
                 for (const ch of this.classicChannels.getAll()) {
                     const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
                     const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
+                    const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
                     const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
                     const modelChanged = oldModels.get(ch.instanceName) !== newModel;
-                    if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged)) {
+                    const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
+                    if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
                         this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
                         await this.stopInstance(ch.instanceName).catch(() => { });
                         // Small delay to let tmux window clean up
                         await new Promise(r => setTimeout(r, 2000));
-                        await this.startClassicInstance(ch.instanceName, newBackend, this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), newModel).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to restart classic instance"));
+                        await this.startClassicInstance(ch.instanceName, newBackend, this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), newModel, newAutoPause).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to restart classic instance"));
                     }
                 }
             }
@@ -858,7 +975,7 @@ export class FleetManager {
                 let idx = 0;
                 while (idx < channels.length) {
                     const batch = channels.slice(idx, idx + concurrency);
-                    await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
+                    await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
                     idx += concurrency;
                 }
             }
@@ -2375,7 +2492,7 @@ export class FleetManager {
                     type: "fleet_schedule_trigger",
                     payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
                     meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
-                });
+                }, { waitForIdle: true });
                 // A scheduled trigger also puts the instance to work — show a cancel button.
                 void this.sendCancelButton(target);
                 return true;
@@ -2943,6 +3060,8 @@ export class FleetManager {
         }
     }
     startStatuslineWatcher(name) {
+        if (this.lifecycle.isPaused(name))
+            return;
         this.statuslineWatcher.watch(name);
     }
     stopStatuslineWatcher(name) {
@@ -4019,7 +4138,7 @@ When users create specialized instances, suggest these configurations:
         await pending.complete(warning ? `${warning}\n\n${reply}` : reply, pending.messageId);
     }
     /** Start a classic channel instance with lightweight config */
-    async startClassicInstance(instanceName, backend, preTaskCommand, model) {
+    async startClassicInstance(instanceName, backend, preTaskCommand, model, autoPauseAfter) {
         if (this.daemons.has(instanceName))
             return;
         const workDir = join(getAgendHome(), "workspaces", instanceName);
@@ -4031,6 +4150,7 @@ When users create specialized instances, suggest these configurations:
             lightweight: true,
             ...(backend ? { backend } : {}),
             ...(model ? { model } : {}),
+            ...(autoPauseAfter !== undefined ? { auto_pause_after: autoPauseAfter } : {}),
             ...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
         };
         const topicMode = this.fleetConfig?.channel?.mode === "topic";
@@ -4045,6 +4165,7 @@ When users create specialized instances, suggest these configurations:
         if (!classicChannels)
             return "Classic channel manager not initialized.";
         const instanceName = classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
+        clearPausedMarker(this.getInstanceDir(instanceName));
         const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
         classicChannels.register(channelId, adapterId, instanceName, channelName || channelId, userId, selectedBackend);
         // Bind this classic instance to the bot that started it (authoritative), so
@@ -4052,7 +4173,7 @@ When users create specialized instances, suggest these configurations:
         // also sees the channel's messages.
         if (adapterId)
             this.bindInstanceAdapter(instanceName, adapterId);
-        await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model));
+        await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
         this.reregisterClassicChannels();
         // Auto-enable collab for Discord classic channels (TG uses @mention directly without collab mode)
         if (guildId && !classicChannels.isCollab(channelId, adapterId)) {
@@ -4070,6 +4191,7 @@ When users create specialized instances, suggest these configurations:
             return t("classic.no_agent");
         this.instanceWorldBinding.delete(ch.instanceName);
         await this.stopInstance(ch.instanceName).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to stop classic instance"));
+        clearPausedMarker(this.getInstanceDir(ch.instanceName));
         this.reregisterClassicChannels();
         this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
         return t("classic.stopped");
@@ -4410,7 +4532,7 @@ When users create specialized instances, suggest these configurations:
                 let idx = 0;
                 while (idx < channels.length) {
                     const batch = channels.slice(idx, idx + concurrency);
-                    await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
+                    await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
                     idx += concurrency;
                 }
             }