@songsid/agend 2.1.5-beta.19 → 2.1.5-beta.20

package/dist/fleet-context.d.ts

@@ -5,6 +5,7 @@ import type { Scheduler } from "./scheduler/index.js";
 import type { Logger } from "./logger.js";
 import type { CostGuard } from "./cost-guard.js";
 import type { ClassicChannelManager } from "./classic-channel-manager.js";
+import type { ExplicitInstanceRemoval } from "./instance-removal.js";
 export type RouteTarget = {
     kind: "instance";
     name: string;
@@ -88,7 +89,13 @@ export interface FleetContext {
     saveFleetConfig(): void;
     getInstanceDir(name: string): string;
     createForumTopic(topicName: string, adapterId?: string): Promise<number | string>;
-    removeInstance(name: string): Promise<void>;
+    removeInstance(name: string, authorization: ExplicitInstanceRemoval): Promise<void>;
+    /** Quarantine a provider-confirmed missing topic without deleting user data. */
+    quarantineMissingTopic(threadId: string, target: RouteTarget, evidence: {
+        source: "provider-event" | "provider-probe";
+        adapterId?: string;
+        generation?: number;
+    }): void;
     getAdapterStates?(): Map<string, {
         status: string;
         retryCount: number;

package/dist/fleet-context.js.map

@@ -1 +1 @@
-{"version":3,"file":"fleet-context.js","sourceRoot":"","sources":["../src/fleet-context.ts"],"names":[],"mappings":"AAgCA,MAAM,UAAU,sBAAsB,CAAC,MAAmB;IACxD,OAAO,MAAM,CAAC,IAAI,KAAK,UAAU,CAAC;AACpC,CAAC"}
+{"version":3,"file":"fleet-context.js","sourceRoot":"","sources":["../src/fleet-context.ts"],"names":[],"mappings":"AAiCA,MAAM,UAAU,sBAAsB,CAAC,MAAmB;IACxD,OAAO,MAAM,CAAC,IAAI,KAAK,UAAU,CAAC;AACpC,CAAC"}

package/dist/fleet-manager.d.ts

@@ -19,6 +19,7 @@ import { type OutboundContext } from "./outbound-handlers.js";
 import { type RawConfigPatch } from "./settings-api.js";
 import { type AgentEndpointContext } from "./agent-endpoint.js";
 import { ClassicChannelManager } from "./classic-channel-manager.js";
+import { type ExplicitInstanceRemoval } from "./instance-removal.js";
 import type { InstanceState } from "./backend/types.js";
 import { StormWindow } from "./storm-window.js";
 import { SpawnGate } from "./spawn-gate.js";
@@ -134,6 +135,10 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private reloadPending;
     /** A running reconciliation; only one may mutate lifecycle/config state at a time. */
     private reconcileInFlight;
+    /** Topology checks are serialized separately from config reconciliation. */
+    private topicCleanupInFlight;
+    private topicCleanupGeneration;
+    private topicProbeWarnings;
     logger: Logger;
     private topicCommands;
     sessionRegistry: Map<string, string>;
@@ -759,8 +764,31 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private sessionPruneTimer;
     private classicReloadTimer;
     private botUserId;
-    /** Periodically check if bound topics still exist */
+    /** Periodically check if bound topics still exist. Never deletes user data. */
     private startTopicCleanupPoller;
+    /** Coalesce timer ticks; an outage must not create overlapping destructive-looking scans. */
+    private scheduleTopicCleanup;
+    private confirmedProbeFence;
+    private sameProbeFence;
+    private warnTopicProbeUnknown;
+    /** One fixed-snapshot topology pass. Automatic evidence can only quarantine. */
+    private runTopicCleanup;
+    /**
+     * A gateway event is only a hint. Confirm it through the passive REST probe;
+     * reconnecting gateways have emitted false channelDelete events in practice.
+     */
+    private handleProviderTopicClosed;
+    private bindTopicClosedHandler;
+    /**
+     * Remove only the volatile route. The instance config, daemon, schedules,
+     * teams, metadata directory, and working tree remain untouched until an
+     * authenticated explicit deletion is requested.
+     */
+    quarantineMissingTopic(threadId: string, target: RouteTarget, evidence: {
+        source: "provider-event" | "provider-probe";
+        adapterId?: string;
+        generation?: number;
+    }): void;
     /**
      * Patch only values changed in the effective config into the original YAML
      * document. Unknown keys and comments remain untouched; redundant
@@ -771,7 +799,7 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private slimFleetConfigAtStartup;
     private writeFleetConfigBackup;
     private patchFleetDocument;
-    removeInstance(name: string): Promise<void>;
+    removeInstance(name: string, authorization: ExplicitInstanceRemoval): Promise<void>;
     startStatuslineWatcher(name: string): void;
     stopStatuslineWatcher(name: string): void;
     reactMessageStatus(instanceName: string, chatId: string, messageId: string, emoji: string): void;

package/dist/fleet-manager.js

@@ -55,6 +55,7 @@ import { handleSettingsRequest } from "./settings-api.js";
 import { setLocale, detectLocale, getLocale, t } from "./locale.js";
 import { handleAgentRequest } from "./agent-endpoint.js";
 import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
+import { assertExplicitInstanceRemoval } from "./instance-removal.js";
 import { validateFleetConfig } from "./config-validator.js";
 import { readLastInboundAt } from "./daemon.js";
 import { clearPausedMarker } from "./pause-marker.js";
@@ -316,6 +317,10 @@ export class FleetManager {
     reloadPending = false;
     /** A running reconciliation; only one may mutate lifecycle/config state at a time. */
     reconcileInFlight = null;
+    /** Topology checks are serialized separately from config reconciliation. */
+    topicCleanupInFlight = null;
+    topicCleanupGeneration = 0;
+    topicProbeWarnings = new Map();
     logger = createLogger("info");
     topicCommands;
     // sessionName → instanceName mapping for external sessions
@@ -3018,10 +3023,11 @@ export class FleetManager {
             accessManager,
             inboxDir,
         });
-        const world = new AdapterWorld(adapterId, this.adapter, accessManager, channelConfig);
+        const adapter = this.adapter;
+        const world = new AdapterWorld(adapterId, adapter, accessManager, channelConfig);
         this.worlds.set(adapterId, world);
-        this.adapters.set(adapterId, this.adapter);
-        this.bindAdapterHealth(this.adapter, adapterId);
+        this.adapters.set(adapterId, adapter);
+        this.bindAdapterHealth(adapter, adapterId);
         this.adapter.on("message", safeHandler(async (msg) => {
             await this.handleInboundMessage(msg);
         }, this.logger, "adapter.message"));
@@ -3066,12 +3072,7 @@ export class FleetManager {
                 return;
             }
         }, this.logger, "adapter.callback_query"));
-        this.adapter.on("topic_closed", safeHandler(async (data) => {
-            // Skip unbind if we archived this topic ourselves
-            if (this.topicArchiver.isArchived(data.threadId))
-                return;
-            await this.topicCommands.handleTopicDeleted(data.threadId);
-        }, this.logger, "adapter.topic_closed"));
+        this.bindTopicClosedHandler(adapter, adapterId, "adapter.topic_closed");
         // Handle classic bot slash commands (/start, /stop, /chat, /compact, /save, /load)
         this.adapter.on("slash_command", safeHandler(async (data) => {
             if (data.command === "start") {
@@ -3462,11 +3463,7 @@ export class FleetManager {
                 return;
             }
         }, this.logger, `adapter[${adapterId}].callback_query`));
-        adapter.on("topic_closed", safeHandler(async (data) => {
-            if (this.topicArchiver.isArchived(data.threadId))
-                return;
-            await this.topicCommands.handleTopicDeleted(data.threadId);
-        }, this.logger, `adapter[${adapterId}].topic_closed`));
+        this.bindTopicClosedHandler(adapter, adapterId, `adapter[${adapterId}].topic_closed`);
         // Slash commands: classic bot + admin commands
         adapter.on("slash_command", safeHandler(async (data) => {
             if (data.command === "start") {
@@ -5456,26 +5453,176 @@ export class FleetManager {
     sessionPruneTimer = null;
     classicReloadTimer = null;
     botUserId;
-    /** Periodically check if bound topics still exist */
+    /** Periodically check if bound topics still exist. Never deletes user data. */
     startTopicCleanupPoller() {
-        this.topicCleanupTimer = setInterval(async () => {
-            if (!this.fleetConfig?.channel?.group_id || !this.adapter?.topicExists)
+        if (this.topicCleanupTimer)
+            clearInterval(this.topicCleanupTimer);
+        const generation = ++this.topicCleanupGeneration;
+        this.topicCleanupTimer = setInterval(() => { void this.scheduleTopicCleanup(generation); }, 5 * 60_000);
+    }
+    /** Coalesce timer ticks; an outage must not create overlapping destructive-looking scans. */
+    scheduleTopicCleanup(generation = this.topicCleanupGeneration) {
+        if (this.topicCleanupInFlight)
+            return this.topicCleanupInFlight;
+        const run = this.runTopicCleanup(generation).finally(() => {
+            if (this.topicCleanupInFlight === run)
+                this.topicCleanupInFlight = null;
+        });
+        this.topicCleanupInFlight = run;
+        return run;
+    }
+    confirmedProbeFence(adapterId, adapter) {
+        const health = adapter.getHealthSnapshot?.();
+        if (health) {
+            return health.status === "connected" && health.isReady
+                ? { generation: health.generation }
+                : null;
+        }
+        return this.adapterState.get(adapterId)?.status === "connected" ? {} : null;
+    }
+    sameProbeFence(adapterId, adapter, before, result) {
+        const after = this.confirmedProbeFence(adapterId, adapter);
+        if (!after)
+            return false;
+        if (before.generation !== after.generation)
+            return false;
+        return result?.generation === undefined || result.generation === after.generation;
+    }
+    warnTopicProbeUnknown(instanceName, threadId, adapterId, reason) {
+        const key = `${adapterId ?? "unbound"}:${reason}`;
+        const now = Date.now();
+        const last = this.topicProbeWarnings.get(key) ?? 0;
+        if (now - last < FleetManager.FLEET_ERROR_THROTTLE_MS)
+            return;
+        this.topicProbeWarnings.set(key, now);
+        this.logger.error({ instanceName, threadId, adapterId, reason }, "Topic presence could not be confirmed — retaining instance and all data");
+        this.notifyFleetError(t("fleet.topic_probe_unknown", instanceName, adapterId ?? "unbound"));
+    }
+    /** One fixed-snapshot topology pass. Automatic evidence can only quarantine. */
+    async runTopicCleanup(generation) {
+        if (generation !== this.topicCleanupGeneration || this.shuttingDown)
+            return;
+        const snapshot = [...this.routing.entries()].filter(([, target]) => isProbeableRouteTarget(target));
+        const missing = [];
+        for (const [threadId, target] of snapshot) {
+            if (generation !== this.topicCleanupGeneration || this.shuttingDown)
                 return;
-            for (const [threadId, target] of this.routing.entries()) {
-                try {
-                    if (!isProbeableRouteTarget(target)) {
-                        continue;
-                    }
-                    const exists = await this.adapter.topicExists(threadId);
-                    if (!exists) {
-                        await this.topicCommands.handleTopicDeleted(threadId);
-                    }
-                }
-                catch (err) {
-                    this.logger.debug({ err, threadId }, "Topic existence check failed");
-                }
+            // The route may have been replaced while an earlier probe was in flight.
+            if (this.routing.resolve(threadId) !== target)
+                continue;
+            const adapterId = this.getInstanceAdapterId(target.name);
+            const adapter = adapterId ? this.adapters.get(adapterId) : undefined;
+            if (!adapterId || !adapter?.probeTopicPresence) {
+                this.warnTopicProbeUnknown(target.name, threadId, adapterId, "owner-adapter-unavailable");
+                continue;
+            }
+            const before = this.confirmedProbeFence(adapterId, adapter);
+            if (!before) {
+                this.warnTopicProbeUnknown(target.name, threadId, adapterId, "owner-adapter-not-ready");
+                continue;
             }
-        }, 5 * 60_000);
+            let result;
+            try {
+                result = await adapter.probeTopicPresence(threadId);
+            }
+            catch {
+                result = { status: "unknown", reason: "provider-probe-threw" };
+            }
+            if (generation !== this.topicCleanupGeneration || this.shuttingDown)
+                return;
+            if (!this.sameProbeFence(adapterId, adapter, before, result)) {
+                this.warnTopicProbeUnknown(target.name, threadId, adapterId, "owner-adapter-generation-changed");
+                continue;
+            }
+            if (result.status === "unknown") {
+                this.warnTopicProbeUnknown(target.name, threadId, adapterId, result.reason);
+            }
+            else if (result.status === "missing") {
+                missing.push({ threadId, target, adapterId, adapter, generation: result.generation });
+            }
+        }
+        if (generation !== this.topicCleanupGeneration || this.shuttingDown || missing.length === 0)
+            return;
+        if (missing.length > 1) {
+            this.logger.error({ missing: missing.map(item => ({ instanceName: item.target.name, threadId: item.threadId, adapterId: item.adapterId })) }, "Multiple topics appeared missing in one pass — treating topology evidence as untrusted and retaining all data");
+            this.notifyFleetError(t("fleet.topic_probe_bulk", missing.length));
+            return;
+        }
+        const item = missing[0];
+        if (this.routing.resolve(item.threadId) !== item.target
+            || this.getInstanceAdapterId(item.target.name) !== item.adapterId
+            || this.adapters.get(item.adapterId) !== item.adapter)
+            return;
+        const current = this.confirmedProbeFence(item.adapterId, item.adapter);
+        if (!current || (item.generation !== undefined && current.generation !== item.generation)) {
+            this.warnTopicProbeUnknown(item.target.name, item.threadId, item.adapterId, "owner-adapter-changed-before-action");
+            return;
+        }
+        this.topicCommands.handleTopicDeleted(item.threadId, {
+            source: "provider-probe",
+            adapterId: item.adapterId,
+            generation: item.generation,
+        });
+    }
+    /**
+     * A gateway event is only a hint. Confirm it through the passive REST probe;
+     * reconnecting gateways have emitted false channelDelete events in practice.
+     */
+    async handleProviderTopicClosed(threadId, adapterId, adapter) {
+        const target = this.routing.resolve(threadId);
+        if (!target || !isProbeableRouteTarget(target))
+            return;
+        if (this.getInstanceAdapterId(target.name) !== adapterId)
+            return;
+        if (this.adapters.get(adapterId) !== adapter)
+            return;
+        const before = this.confirmedProbeFence(adapterId, adapter);
+        if (!before || !adapter.probeTopicPresence) {
+            this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-from-unready-adapter");
+            return;
+        }
+        let result;
+        try {
+            result = await adapter.probeTopicPresence(threadId);
+        }
+        catch {
+            result = { status: "unknown", reason: "provider-probe-threw" };
+        }
+        if (this.routing.resolve(threadId) !== target
+            || this.getInstanceAdapterId(target.name) !== adapterId
+            || this.adapters.get(adapterId) !== adapter
+            || !this.sameProbeFence(adapterId, adapter, before, result)) {
+            this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-generation-changed");
+            return;
+        }
+        if (result.status !== "missing") {
+            this.warnTopicProbeUnknown(target.name, threadId, adapterId, result.status === "unknown" ? result.reason : "topic-close-not-confirmed-missing");
+            return;
+        }
+        this.topicCommands.handleTopicDeleted(threadId, {
+            source: "provider-event",
+            adapterId,
+            generation: result.generation,
+        });
+    }
+    bindTopicClosedHandler(adapter, adapterId, label) {
+        adapter.on("topic_closed", safeHandler(async (data) => {
+            if (this.topicArchiver.isArchived(data.threadId))
+                return;
+            await this.handleProviderTopicClosed(data.threadId, adapterId, adapter);
+        }, this.logger, label));
+    }
+    /**
+     * Remove only the volatile route. The instance config, daemon, schedules,
+     * teams, metadata directory, and working tree remain untouched until an
+     * authenticated explicit deletion is requested.
+     */
+    quarantineMissingTopic(threadId, target, evidence) {
+        if (!isProbeableRouteTarget(target) || this.routing.resolve(threadId) !== target)
+            return;
+        this.routing.unregister(threadId);
+        this.logger.error({ instanceName: target.name, threadId, ...evidence }, "Topic is confirmed missing — route quarantined; instance configuration and user data were retained");
+        this.notifyFleetError(t("fleet.topic_quarantined", target.name, threadId));
     }
     /**
      * Patch only values changed in the effective config into the original YAML
@@ -5628,7 +5775,8 @@ export class FleetManager {
             }
         }
     }
-    async removeInstance(name) {
+    async removeInstance(name, authorization) {
+        assertExplicitInstanceRemoval(authorization);
         // Drop cached pane context — the map is keyed by instance name and nothing
         // else evicted deleted entries, so it grew for the life of the process.
         forgetInstanceContext(name);
@@ -5654,7 +5802,7 @@ export class FleetManager {
                 }
             }
         }
-        await this.lifecycle.remove(name);
+        await this.lifecycle.remove(name, authorization);
         // Clean up per-instance tracking maps so they don't grow unbounded
         // as instances are created and deleted over the lifetime of the fleet.
         this.lastActivity.delete(name);
@@ -9995,6 +10143,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             clearInterval(this.topicCleanupTimer);
             this.topicCleanupTimer = null;
         }
+        this.topicCleanupGeneration++;
         if (this.sessionPruneTimer) {
             clearInterval(this.sessionPruneTimer);
             this.sessionPruneTimer = null;